From 9c6351fa643f76a7cf36eef3b80cff9bf5880463 Mon Sep 17 00:00:00 2001 From: mark9white Date: Sun, 12 Apr 2020 19:44:25 +0100 Subject: [PATCH] feat: Allow step restart on workflow retry. Closes #2334 (#2431) --- api/openapi-spec/swagger.json | 7 + cmd/argo/commands/retry.go | 22 +- pkg/apiclient/workflow/workflow.pb.go | 242 +++++++++++++------ pkg/apiclient/workflow/workflow.proto | 2 + pkg/apiclient/workflow/workflow.swagger.json | 7 + server/workflow/workflow_server.go | 2 +- test/e2e/cli_test.go | 33 ++- test/e2e/testdata/retry-test.yaml | 31 +++ workflow/controller/operator.go | 25 +- workflow/util/util.go | 74 +++++- workflow/util/util_test.go | 4 +- 11 files changed, 349 insertions(+), 100 deletions(-) create mode 100644 test/e2e/testdata/retry-test.yaml diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 2e6c75daf658..a390d08dcf88 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -6147,6 +6147,13 @@ }, "namespace": { "type": "string" + }, + "nodeFieldSelector": { + "type": "string" + }, + "restartSuccesful": { + "type": "boolean", + "format": "boolean" } } }, diff --git a/cmd/argo/commands/retry.go b/cmd/argo/commands/retry.go index 1df59858e0da..665fb5abb00f 100644 --- a/cmd/argo/commands/retry.go +++ b/cmd/argo/commands/retry.go @@ -1,16 +1,25 @@ package commands import ( + "log" + "github.com/argoproj/pkg/errors" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/fields" "github.com/argoproj/argo/cmd/argo/commands/client" workflowpkg "github.com/argoproj/argo/pkg/apiclient/workflow" ) +type retryOps struct { + nodeFieldSelector string // --node-field-selector + restartSuccessful bool // --restart-successful +} + func NewRetryCommand() *cobra.Command { var ( cliSubmitOpts cliSubmitOpts + retryOps retryOps ) var command = &cobra.Command{ Use: "retry [WORKFLOW...]", @@ -20,10 +29,17 @@ func NewRetryCommand() *cobra.Command { serviceClient := apiClient.NewWorkflowServiceClient() namespace := client.Namespace() + selector, err := fields.ParseSelector(retryOps.nodeFieldSelector) + if err != nil { + log.Fatalf("Unable to parse node field selector '%s': %s", retryOps.nodeFieldSelector, err) + } + for _, name := range args { wf, err := serviceClient.RetryWorkflow(ctx, &workflowpkg.WorkflowRetryRequest{ - Name: name, - Namespace: namespace, + Name: name, + Namespace: namespace, + RestartSuccesful: retryOps.restartSuccessful, + NodeFieldSelector: selector.String(), }) if err != nil { errors.CheckError(err) @@ -37,5 +53,7 @@ func NewRetryCommand() *cobra.Command { command.Flags().StringVarP(&cliSubmitOpts.output, "output", "o", "", "Output format. One of: name|json|yaml|wide") command.Flags().BoolVarP(&cliSubmitOpts.wait, "wait", "w", false, "wait for the workflow to complete") command.Flags().BoolVar(&cliSubmitOpts.watch, "watch", false, "watch the workflow until it completes") + command.Flags().BoolVar(&retryOps.restartSuccessful, "restart-successful", false, "indicates to restart succesful nodes matching the --node-field-selector") + command.Flags().StringVar(&retryOps.nodeFieldSelector, "node-field-selector", "", "selector of nodes to reset, eg: --node-field-selector inputs.paramaters.myparam.value=abc") return command } diff --git a/pkg/apiclient/workflow/workflow.pb.go b/pkg/apiclient/workflow/workflow.pb.go index 66d06f7d272e..4b8f467330c9 100644 --- a/pkg/apiclient/workflow/workflow.pb.go +++ b/pkg/apiclient/workflow/workflow.pb.go @@ -316,6 +316,8 @@ func (m *WorkflowResubmitRequest) GetMemoized() bool { type WorkflowRetryRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + RestartSuccesful bool `protobuf:"varint,3,opt,name=restartSuccesful,proto3" json:"restartSuccesful,omitempty"` + NodeFieldSelector string `protobuf:"bytes,4,opt,name=nodeFieldSelector,proto3" json:"nodeFieldSelector,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -368,6 +370,20 @@ func (m *WorkflowRetryRequest) GetNamespace() string { return "" } +func (m *WorkflowRetryRequest) GetRestartSuccesful() bool { + if m != nil { + return m.RestartSuccesful + } + return false +} + +func (m *WorkflowRetryRequest) GetNodeFieldSelector() string { + if m != nil { + return m.NodeFieldSelector + } + return "" +} + type WorkflowResumeRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` @@ -1031,80 +1047,82 @@ func init() { } var fileDescriptor_1f6bb75f9e833cb6 = []byte{ - // 1159 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x98, 0x5b, 0x6f, 0xdc, 0x44, - 0x14, 0xc7, 0x35, 0x69, 0xc8, 0x65, 0x72, 0x69, 0x3b, 0x40, 0x59, 0x59, 0x69, 0x9a, 0x0e, 0x04, - 0x42, 0x9a, 0xd8, 0xd9, 0x24, 0x40, 0xa9, 0x54, 0xa4, 0x92, 0x94, 0x00, 0x5a, 0x41, 0xe5, 0x80, - 0x50, 0x79, 0x73, 0xbc, 0x27, 0x8e, 0x1b, 0xdb, 0x63, 0xec, 0xd9, 0x8d, 0x96, 0x2a, 0x20, 0x78, - 0x44, 0x08, 0x1e, 0x78, 0x41, 0xf0, 0x82, 0xb8, 0x48, 0x48, 0x48, 0x5c, 0xbe, 0x05, 0x8f, 0x48, - 0x7c, 0x01, 0x14, 0xf1, 0x41, 0xd0, 0x8c, 0xed, 0xb1, 0x9d, 0x5d, 0x16, 0x13, 0x23, 0xfa, 0x36, - 0xb3, 0xb3, 0x73, 0xfe, 0xbf, 0x39, 0x97, 0x99, 0x23, 0xe3, 0xc5, 0xf0, 0xd0, 0x31, 0xac, 0xd0, - 0xb5, 0x3d, 0x17, 0x02, 0x6e, 0x1c, 0xb1, 0xe8, 0x70, 0xdf, 0x63, 0x47, 0x6a, 0xa0, 0x87, 0x11, - 0xe3, 0x8c, 0x4c, 0x64, 0x73, 0x6d, 0xd5, 0x71, 0xf9, 0x41, 0x67, 0x4f, 0xb7, 0x99, 0x6f, 0x38, - 0xcc, 0x61, 0x86, 0xfc, 0xc3, 0x5e, 0x67, 0x5f, 0xce, 0xe4, 0x44, 0x8e, 0x92, 0x8d, 0xda, 0x9c, - 0xc3, 0x98, 0xe3, 0x81, 0x90, 0x30, 0xac, 0x20, 0x60, 0xdc, 0xe2, 0x2e, 0x0b, 0xe2, 0x74, 0x75, - 0xf3, 0xf0, 0x7a, 0xac, 0xbb, 0x4c, 0xac, 0xfa, 0x96, 0x7d, 0xe0, 0x06, 0x10, 0xf5, 0x8c, 0x94, - 0x28, 0x36, 0x7c, 0xe0, 0x96, 0xd1, 0x6d, 0x1a, 0x0e, 0x04, 0x10, 0x59, 0x1c, 0xda, 0xe9, 0xae, - 0xad, 0x02, 0x82, 0x15, 0x49, 0xd1, 0x7b, 0x72, 0x90, 0x6f, 0x55, 0xe7, 0xe8, 0x36, 0x2d, 0x2f, - 0x3c, 0xb0, 0xfa, 0x8d, 0xd0, 0x5c, 0xda, 0xb0, 0x59, 0x04, 0x03, 0x84, 0xe8, 0x4f, 0x23, 0xf8, - 0xd1, 0xb7, 0x52, 0x4b, 0x5b, 0x11, 0x58, 0x1c, 0x4c, 0x78, 0xa7, 0x03, 0x31, 0x27, 0x73, 0x78, - 0x32, 0xb0, 0x7c, 0x88, 0x43, 0xcb, 0x86, 0x06, 0x5a, 0x40, 0x4b, 0x93, 0x66, 0xfe, 0x03, 0xb9, - 0x8b, 0x95, 0xbf, 0x1a, 0x23, 0x0b, 0x68, 0x69, 0x6a, 0xfd, 0xa6, 0x9e, 0x33, 0xeb, 0x19, 0xb3, - 0x1c, 0xe8, 0xe1, 0xa1, 0xa3, 0x0b, 0x66, 0x5d, 0xb9, 0x3c, 0x63, 0xd6, 0x33, 0x6d, 0x53, 0x99, - 0x23, 0xf3, 0x18, 0xbb, 0x41, 0xcc, 0xad, 0xc0, 0x86, 0x57, 0xb6, 0x1b, 0xe7, 0xa4, 0x72, 0xe1, - 0x17, 0x42, 0xf1, 0x74, 0x0c, 0x51, 0x17, 0xa2, 0xed, 0xa8, 0x67, 0x76, 0x82, 0xc6, 0xe8, 0x02, - 0x5a, 0x9a, 0x30, 0x4b, 0xbf, 0x91, 0xbb, 0x78, 0xc6, 0x96, 0xa7, 0x79, 0x3d, 0x94, 0xc1, 0x68, - 0x3c, 0x24, 0x19, 0x37, 0xf4, 0xc4, 0x25, 0x7a, 0x31, 0x1a, 0x39, 0x9e, 0x88, 0x86, 0xde, 0x6d, - 0xea, 0x5b, 0xc5, 0xad, 0x66, 0xd9, 0x12, 0xfd, 0x19, 0x61, 0x92, 0x51, 0xef, 0x00, 0xcf, 0xdc, - 0x45, 0xf0, 0xa8, 0xf0, 0x4e, 0xea, 0x29, 0x39, 0x2e, 0xbb, 0x70, 0xe4, 0xb4, 0x0b, 0xef, 0x60, - 0xec, 0x00, 0xcf, 0x00, 0xcf, 0x49, 0xc0, 0xb5, 0x6a, 0x80, 0x3b, 0x6a, 0x9f, 0x59, 0xb0, 0x41, - 0x2e, 0xe1, 0xb1, 0x7d, 0x17, 0xbc, 0x76, 0x2c, 0x7d, 0x32, 0x69, 0xa6, 0x33, 0xfa, 0x15, 0xc2, - 0x0f, 0x67, 0xc8, 0x2d, 0x37, 0xe6, 0xd5, 0x42, 0xbc, 0x8b, 0xa7, 0x3c, 0x37, 0x56, 0x80, 0x49, - 0x94, 0x9b, 0xd5, 0x00, 0x5b, 0xf9, 0x46, 0xb3, 0x68, 0xa5, 0x80, 0x78, 0xae, 0x84, 0xe8, 0xe0, - 0xc7, 0x54, 0x2a, 0x40, 0xdc, 0xd9, 0xf3, 0xdd, 0x1a, 0x9e, 0xd5, 0xf0, 0x84, 0x0f, 0x3e, 0x73, - 0xdf, 0x85, 0xb6, 0x94, 0x99, 0x30, 0xd5, 0x9c, 0xbe, 0x8c, 0x1f, 0xc9, 0x85, 0x78, 0xd4, 0x3b, - 0xb3, 0x0a, 0x3d, 0xca, 0x2b, 0x47, 0x20, 0xfb, 0x70, 0x76, 0xe0, 0x15, 0x7c, 0x31, 0x60, 0x6d, - 0x78, 0x49, 0xf8, 0x62, 0x17, 0x3c, 0xb0, 0x39, 0x8b, 0x52, 0x07, 0xf5, 0x2f, 0xd0, 0x16, 0x6e, - 0x64, 0xc2, 0x6f, 0x40, 0xe4, 0xbb, 0x41, 0xa1, 0x6a, 0xff, 0xfd, 0x31, 0x3e, 0x2d, 0x24, 0xc7, - 0x2e, 0x67, 0xe1, 0xff, 0x74, 0x0a, 0xd2, 0xc0, 0xe3, 0x3e, 0xc4, 0xb1, 0xe5, 0x40, 0x9a, 0xad, - 0xd9, 0x94, 0xbe, 0x8a, 0x2f, 0x29, 0xa0, 0x4e, 0x1c, 0x42, 0xd0, 0x3e, 0xfb, 0xe9, 0xbe, 0x29, - 0x54, 0x6b, 0x8b, 0x39, 0x67, 0x3f, 0x5c, 0x03, 0x8f, 0x87, 0xac, 0xfd, 0x9a, 0xd8, 0x94, 0x1c, - 0x29, 0x9b, 0x92, 0x5b, 0x18, 0x7b, 0xcc, 0xc9, 0xca, 0x64, 0x54, 0x96, 0xc9, 0xd5, 0x42, 0x99, - 0xe8, 0xe2, 0xee, 0x15, 0x45, 0x71, 0x87, 0xb5, 0x5b, 0xea, 0x8f, 0x66, 0x61, 0x13, 0xfd, 0x16, - 0xe5, 0xb9, 0xb4, 0x0d, 0x1e, 0xd4, 0x88, 0xa7, 0xb8, 0xfa, 0xda, 0xd2, 0x44, 0xf9, 0x66, 0xa9, - 0x78, 0xf5, 0x6d, 0x17, 0xb7, 0x9a, 0x65, 0x4b, 0xb4, 0x91, 0x07, 0x26, 0xa3, 0x8c, 0x43, 0x16, - 0xc4, 0x40, 0x3f, 0x12, 0x07, 0xb0, 0xb8, 0x7d, 0x90, 0xad, 0xc7, 0x0f, 0xee, 0x8e, 0xa1, 0xef, - 0xe7, 0x21, 0x97, 0x4c, 0xb7, 0xbb, 0x10, 0x48, 0x4f, 0xf2, 0x5e, 0xa8, 0x3c, 0x29, 0xc6, 0xe4, - 0x4d, 0x3c, 0xc6, 0xf6, 0xee, 0x81, 0xcd, 0xff, 0x9b, 0x37, 0x2c, 0x35, 0x46, 0x5f, 0xc0, 0x13, - 0x2d, 0xe6, 0xdc, 0x0e, 0x78, 0xd4, 0x13, 0x79, 0x63, 0xb3, 0x80, 0x43, 0xc0, 0x53, 0xe5, 0x6c, - 0x5a, 0xcc, 0xa8, 0x91, 0x52, 0x46, 0xd1, 0x4f, 0x4a, 0xf7, 0x75, 0xc0, 0x1f, 0xf4, 0x93, 0xbc, - 0xfe, 0xf1, 0x05, 0x7c, 0x5e, 0x95, 0x24, 0x44, 0x5d, 0xd7, 0x06, 0xf2, 0x05, 0xc2, 0xb3, 0xc9, - 0x43, 0x99, 0xad, 0x90, 0x2b, 0xb9, 0xb5, 0x81, 0x3d, 0x85, 0x56, 0x0f, 0x88, 0x2e, 0x7d, 0xf8, - 0xfb, 0x9f, 0x9f, 0x8d, 0x50, 0x7a, 0x59, 0xb6, 0x34, 0xdd, 0xa6, 0xea, 0x81, 0x62, 0xe3, 0xbe, - 0xf2, 0xc3, 0xf1, 0x0d, 0xb4, 0x4c, 0x3e, 0x47, 0x78, 0x6a, 0x07, 0xb8, 0x22, 0x9b, 0xeb, 0x27, - 0xcb, 0xdf, 0xee, 0xba, 0x58, 0x2b, 0x12, 0xeb, 0x49, 0xf2, 0xc4, 0x50, 0xac, 0x64, 0x7c, 0x2c, - 0xd0, 0x66, 0x44, 0xea, 0xaa, 0x4a, 0x21, 0x97, 0xfb, 0xe1, 0x0a, 0xaf, 0xb4, 0x76, 0xab, 0x16, - 0x9d, 0xb0, 0x44, 0x17, 0x25, 0xe1, 0x15, 0x32, 0xdc, 0x71, 0xe4, 0x3d, 0x3c, 0x5b, 0x2e, 0xe2, - 0x52, 0x44, 0x07, 0x95, 0xb7, 0x36, 0xc0, 0xb1, 0x79, 0xcd, 0xd1, 0x6b, 0x52, 0x77, 0x91, 0x3c, - 0x7e, 0x5a, 0x77, 0x15, 0xc4, 0x7a, 0x49, 0x7d, 0x0d, 0x91, 0x0f, 0x10, 0x9e, 0x4d, 0x2e, 0x96, - 0x61, 0x29, 0x55, 0xba, 0x20, 0xb5, 0x85, 0xbf, 0xff, 0x43, 0x7a, 0x37, 0xa5, 0xe1, 0x59, 0xae, - 0x16, 0x9e, 0xef, 0x10, 0x9e, 0x91, 0x8d, 0x81, 0x42, 0x98, 0xef, 0x57, 0x28, 0x76, 0x0e, 0x75, - 0xb3, 0xe7, 0x19, 0x89, 0x67, 0x68, 0xcb, 0x55, 0xf0, 0x8c, 0x48, 0x28, 0x8b, 0x0c, 0xff, 0x11, - 0xe1, 0x0b, 0x59, 0xa7, 0xa4, 0x50, 0xaf, 0x0e, 0x42, 0x2d, 0x75, 0x53, 0x75, 0x69, 0xaf, 0x4b, - 0xda, 0x75, 0x6d, 0xb5, 0x22, 0x6d, 0x22, 0x2e, 0x80, 0xbf, 0x47, 0x78, 0x36, 0xe9, 0x93, 0x86, - 0x05, 0xb7, 0xd4, 0x49, 0xd5, 0x85, 0x7d, 0x56, 0xc2, 0xae, 0x69, 0xd7, 0x2a, 0xc3, 0xfa, 0x20, - 0x50, 0x7f, 0x40, 0xf8, 0x7c, 0xda, 0x79, 0x28, 0xd6, 0x01, 0x79, 0x56, 0x6e, 0x4e, 0xea, 0xc2, - 0x3e, 0x27, 0x61, 0x9b, 0xda, 0x4a, 0x25, 0xd8, 0x38, 0xd1, 0x16, 0xb4, 0xbf, 0x20, 0x7c, 0x51, - 0xf5, 0x81, 0x8a, 0x97, 0xf6, 0xf3, 0x9e, 0x6e, 0x16, 0xeb, 0x12, 0x3f, 0x2f, 0x89, 0x37, 0x34, - 0xbd, 0x12, 0x31, 0xcf, 0xd4, 0x05, 0xf3, 0xd7, 0x08, 0x4f, 0x8b, 0x66, 0x53, 0xe1, 0x0e, 0xb8, - 0x03, 0x0b, 0xcd, 0x68, 0x5d, 0xd2, 0x4d, 0x49, 0xaa, 0x6b, 0x4f, 0x57, 0xf3, 0x2d, 0x67, 0xa1, - 0x80, 0xfc, 0x12, 0xe1, 0x69, 0xf1, 0xfc, 0x0e, 0x83, 0x2c, 0x3c, 0xcf, 0x75, 0x21, 0x57, 0x25, - 0xe4, 0x53, 0x94, 0x0e, 0x87, 0xf4, 0xdc, 0x40, 0xd6, 0x53, 0x0f, 0x8f, 0x27, 0x0d, 0x65, 0x3c, - 0xe8, 0x75, 0xcb, 0x7b, 0x5d, 0x8d, 0xe4, 0xab, 0x59, 0x57, 0x42, 0x6f, 0x48, 0xad, 0x4d, 0xb2, - 0x5e, 0xc9, 0x21, 0xf7, 0xd3, 0xc6, 0xe4, 0xd8, 0xf0, 0x98, 0xb3, 0x86, 0x5e, 0xbc, 0xf9, 0xeb, - 0xc9, 0x3c, 0xfa, 0xed, 0x64, 0x1e, 0xfd, 0x71, 0x32, 0x8f, 0xde, 0x36, 0xfe, 0xe9, 0x5b, 0xc5, - 0xa9, 0x0f, 0x2f, 0x7b, 0x63, 0xf2, 0xd3, 0xc3, 0xc6, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xff, - 0xd7, 0xe7, 0x76, 0x99, 0x11, 0x00, 0x00, + // 1186 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x98, 0xdd, 0x6e, 0xdc, 0x44, + 0x1b, 0xc7, 0x35, 0x49, 0xdf, 0x26, 0x99, 0x7c, 0x34, 0x9d, 0x17, 0xca, 0xca, 0x4a, 0xd3, 0x74, + 0x20, 0x10, 0xd2, 0xc4, 0xce, 0x26, 0x01, 0x4a, 0xa5, 0x22, 0x95, 0xa4, 0x44, 0xa0, 0x15, 0x54, + 0x0e, 0x08, 0x95, 0x33, 0xc7, 0xfb, 0xc4, 0x71, 0xe3, 0xf5, 0x18, 0xcf, 0xec, 0x46, 0x4b, 0x15, + 0x10, 0x1c, 0x22, 0x04, 0x07, 0x9c, 0x20, 0x38, 0x41, 0x05, 0x24, 0x24, 0x24, 0x3e, 0xee, 0x82, + 0x43, 0x24, 0x6e, 0x00, 0x45, 0x5c, 0x08, 0x9a, 0x59, 0x7b, 0x6c, 0x67, 0xb7, 0x8b, 0x89, 0x11, + 0x3d, 0x9b, 0xf1, 0xec, 0xcc, 0xff, 0x37, 0xcf, 0xd7, 0x3c, 0x5a, 0xbc, 0x18, 0x1d, 0x7a, 0x96, + 0x13, 0xf9, 0x6e, 0xe0, 0x43, 0x28, 0xac, 0x23, 0x16, 0x1f, 0xee, 0x07, 0xec, 0x48, 0x0f, 0xcc, + 0x28, 0x66, 0x82, 0x91, 0xf1, 0x74, 0x6e, 0xac, 0x7a, 0xbe, 0x38, 0x68, 0xef, 0x99, 0x2e, 0x6b, + 0x59, 0x1e, 0xf3, 0x98, 0xa5, 0x7e, 0xb0, 0xd7, 0xde, 0x57, 0x33, 0x35, 0x51, 0xa3, 0xde, 0x46, + 0x63, 0xce, 0x63, 0xcc, 0x0b, 0x40, 0x4a, 0x58, 0x4e, 0x18, 0x32, 0xe1, 0x08, 0x9f, 0x85, 0x3c, + 0x59, 0xdd, 0x3c, 0xbc, 0xce, 0x4d, 0x9f, 0xc9, 0xd5, 0x96, 0xe3, 0x1e, 0xf8, 0x21, 0xc4, 0x5d, + 0x2b, 0x21, 0xe2, 0x56, 0x0b, 0x84, 0x63, 0x75, 0xea, 0x96, 0x07, 0x21, 0xc4, 0x8e, 0x80, 0x66, + 0xb2, 0x6b, 0x2b, 0x87, 0xe0, 0xc4, 0x4a, 0xf4, 0x9e, 0x1a, 0x64, 0x5b, 0xf5, 0x3d, 0x3a, 0x75, + 0x27, 0x88, 0x0e, 0x9c, 0xfe, 0x43, 0x68, 0x26, 0x6d, 0xb9, 0x2c, 0x86, 0x01, 0x42, 0xf4, 0xa7, + 0x11, 0xfc, 0xf8, 0xdb, 0xc9, 0x49, 0x5b, 0x31, 0x38, 0x02, 0x6c, 0x78, 0xb7, 0x0d, 0x5c, 0x90, + 0x39, 0x3c, 0x11, 0x3a, 0x2d, 0xe0, 0x91, 0xe3, 0x42, 0x0d, 0x2d, 0xa0, 0xa5, 0x09, 0x3b, 0xfb, + 0x40, 0xee, 0x62, 0x6d, 0xaf, 0xda, 0xc8, 0x02, 0x5a, 0x9a, 0x5c, 0xbf, 0x69, 0x66, 0xcc, 0x66, + 0xca, 0xac, 0x06, 0x66, 0x74, 0xe8, 0x99, 0x92, 0xd9, 0xd4, 0x26, 0x4f, 0x99, 0xcd, 0x54, 0xdb, + 0xd6, 0xc7, 0x91, 0x79, 0x8c, 0xfd, 0x90, 0x0b, 0x27, 0x74, 0xe1, 0xd5, 0xed, 0xda, 0xa8, 0x52, + 0xce, 0x7d, 0x21, 0x14, 0x4f, 0x71, 0x88, 0x3b, 0x10, 0x6f, 0xc7, 0x5d, 0xbb, 0x1d, 0xd6, 0xce, + 0x2d, 0xa0, 0xa5, 0x71, 0xbb, 0xf0, 0x8d, 0xdc, 0xc5, 0xd3, 0xae, 0xba, 0xcd, 0x1b, 0x91, 0x72, + 0x46, 0xed, 0x7f, 0x8a, 0x71, 0xc3, 0xec, 0x99, 0xc4, 0xcc, 0x7b, 0x23, 0xc3, 0x93, 0xde, 0x30, + 0x3b, 0x75, 0x73, 0x2b, 0xbf, 0xd5, 0x2e, 0x9e, 0x44, 0x7f, 0x46, 0x98, 0xa4, 0xd4, 0x3b, 0x20, + 0x52, 0x73, 0x11, 0x7c, 0x4e, 0x5a, 0x27, 0xb1, 0x94, 0x1a, 0x17, 0x4d, 0x38, 0x72, 0xda, 0x84, + 0x77, 0x30, 0xf6, 0x40, 0xa4, 0x80, 0xa3, 0x0a, 0x70, 0xad, 0x1c, 0xe0, 0x8e, 0xde, 0x67, 0xe7, + 0xce, 0x20, 0x97, 0xf0, 0xf9, 0x7d, 0x1f, 0x82, 0x26, 0x57, 0x36, 0x99, 0xb0, 0x93, 0x19, 0xfd, + 0x1a, 0xe1, 0xff, 0xa7, 0xc8, 0x0d, 0x9f, 0x8b, 0x72, 0x2e, 0xde, 0xc5, 0x93, 0x81, 0xcf, 0x35, + 0x60, 0xcf, 0xcb, 0xf5, 0x72, 0x80, 0x8d, 0x6c, 0xa3, 0x9d, 0x3f, 0x25, 0x87, 0x38, 0x5a, 0x40, + 0xf4, 0xf0, 0x13, 0x3a, 0x14, 0x80, 0xb7, 0xf7, 0x5a, 0x7e, 0x05, 0xcb, 0x1a, 0x78, 0xbc, 0x05, + 0x2d, 0xe6, 0xbf, 0x07, 0x4d, 0x25, 0x33, 0x6e, 0xeb, 0x39, 0x7d, 0x80, 0xf0, 0x63, 0x99, 0x92, + 0x88, 0xbb, 0x67, 0x97, 0x59, 0xc6, 0xb3, 0x31, 0x70, 0xe1, 0xc4, 0x62, 0xb7, 0xed, 0xba, 0xc0, + 0xf7, 0xdb, 0x41, 0x22, 0xd7, 0xf7, 0x9d, 0xac, 0xe0, 0x8b, 0x21, 0x6b, 0xc2, 0x2b, 0xf2, 0xb6, + 0xbb, 0x10, 0x80, 0x2b, 0x58, 0x9c, 0x78, 0xa9, 0x7f, 0x81, 0x1e, 0x65, 0x49, 0x29, 0xad, 0xd1, + 0x82, 0xb3, 0x43, 0x0e, 0x14, 0x1e, 0x7d, 0x98, 0x70, 0x03, 0xd7, 0x52, 0xe1, 0x37, 0x21, 0x6e, + 0xf9, 0x61, 0xae, 0x20, 0xfc, 0x63, 0x6d, 0xfa, 0x59, 0x2e, 0xee, 0x76, 0x05, 0x8b, 0xfe, 0xa3, + 0x5b, 0x90, 0x1a, 0x1e, 0x6b, 0x01, 0xe7, 0x8e, 0x07, 0x89, 0x89, 0xd3, 0x29, 0x7d, 0x0d, 0x5f, + 0xd2, 0x40, 0x6d, 0x1e, 0x41, 0xd8, 0x3c, 0xfb, 0xed, 0xbe, 0xc9, 0x15, 0x82, 0x06, 0xf3, 0xce, + 0x7e, 0xb9, 0x1a, 0x1e, 0x8b, 0x58, 0xf3, 0x75, 0xb9, 0xa9, 0x77, 0xa5, 0x74, 0x4a, 0x6e, 0x61, + 0x1c, 0x30, 0x2f, 0xcd, 0xc0, 0x73, 0x2a, 0x03, 0xaf, 0xe6, 0x32, 0xd0, 0x94, 0x65, 0x5d, 0xe6, + 0xdb, 0x1d, 0xd6, 0x6c, 0xe8, 0x1f, 0xda, 0xb9, 0x4d, 0xf4, 0x5b, 0x94, 0xc5, 0xd2, 0x36, 0x04, + 0x50, 0xc1, 0x9f, 0xb2, 0xaa, 0x36, 0xd5, 0x11, 0xc5, 0xa2, 0x55, 0xb2, 0xaa, 0x6e, 0xe7, 0xb7, + 0xda, 0xc5, 0x93, 0x68, 0x2d, 0x73, 0x4c, 0x4a, 0xc9, 0x23, 0x16, 0x72, 0xa0, 0x1f, 0xcb, 0x0b, + 0x38, 0xc2, 0x3d, 0x48, 0xd7, 0xf9, 0xa3, 0x2b, 0x5f, 0xf4, 0x83, 0xcc, 0xe5, 0x8a, 0xe9, 0x76, + 0x07, 0x42, 0x65, 0x49, 0xd1, 0x8d, 0xb4, 0x25, 0xe5, 0x98, 0xbc, 0x85, 0xcf, 0xb3, 0xbd, 0x7b, + 0xe0, 0x8a, 0x7f, 0xe7, 0x79, 0x4c, 0x0e, 0xa3, 0x2f, 0xe1, 0xf1, 0x06, 0xf3, 0x6e, 0x87, 0x22, + 0xee, 0xca, 0xb8, 0x71, 0x59, 0x28, 0x20, 0x14, 0x89, 0x72, 0x3a, 0xcd, 0x47, 0xd4, 0x48, 0x21, + 0xa2, 0xe8, 0xa7, 0x85, 0xa7, 0x20, 0x14, 0x8f, 0xfa, 0xb5, 0x5f, 0xff, 0x64, 0x16, 0x5f, 0xd0, + 0x29, 0x09, 0x71, 0xc7, 0x77, 0x81, 0x7c, 0x89, 0xf0, 0x4c, 0xef, 0x0d, 0x4e, 0x57, 0xc8, 0x95, + 0xec, 0xb4, 0x81, 0xed, 0x8a, 0x51, 0x0d, 0x88, 0x2e, 0x7d, 0xf4, 0xfb, 0x9f, 0x9f, 0x8f, 0x50, + 0x7a, 0x59, 0x75, 0x4b, 0x9d, 0xba, 0x6e, 0xaf, 0xb8, 0x75, 0x5f, 0xdb, 0xe1, 0xf8, 0x06, 0x5a, + 0x26, 0x5f, 0x20, 0x3c, 0xb9, 0x03, 0x42, 0x93, 0xcd, 0xf5, 0x93, 0x65, 0x6d, 0x41, 0x55, 0xac, + 0x15, 0x85, 0xf5, 0x34, 0x79, 0x6a, 0x28, 0x56, 0x6f, 0x7c, 0x2c, 0xd1, 0xa6, 0x65, 0xe8, 0xea, + 0x4c, 0x21, 0x97, 0xfb, 0xe1, 0x72, 0x0d, 0x80, 0x71, 0xab, 0x12, 0x9d, 0x3c, 0x89, 0x2e, 0x2a, + 0xc2, 0x2b, 0x64, 0xb8, 0xe1, 0xc8, 0xfb, 0x78, 0xa6, 0x98, 0xc4, 0x05, 0x8f, 0x0e, 0x4a, 0x6f, + 0x63, 0x80, 0x61, 0xb3, 0x9c, 0xa3, 0xd7, 0x94, 0xee, 0x22, 0x79, 0xf2, 0xb4, 0xee, 0x2a, 0xc8, + 0xf5, 0x82, 0xfa, 0x1a, 0x22, 0x1f, 0x22, 0x3c, 0xd3, 0x2b, 0x2c, 0xc3, 0x42, 0xaa, 0x50, 0x20, + 0x8d, 0x85, 0x87, 0xff, 0x20, 0xa9, 0x4d, 0x89, 0x7b, 0x96, 0xcb, 0xb9, 0xe7, 0x3b, 0x84, 0xa7, + 0x55, 0xcb, 0xa1, 0x11, 0xe6, 0xfb, 0x15, 0xf2, 0x3d, 0x49, 0xd5, 0xe8, 0x79, 0x4e, 0xe1, 0x59, + 0xc6, 0x72, 0x19, 0x3c, 0x2b, 0x96, 0xca, 0x32, 0xc2, 0x7f, 0x44, 0x78, 0x36, 0x6d, 0xc2, 0x34, + 0xea, 0xd5, 0x41, 0xa8, 0x85, 0x46, 0xad, 0x2a, 0xed, 0x75, 0x45, 0xbb, 0x6e, 0xac, 0x96, 0xa4, + 0xed, 0x89, 0x4b, 0xe0, 0xef, 0x11, 0x9e, 0xe9, 0xf5, 0x49, 0xc3, 0x9c, 0x5b, 0xe8, 0xa4, 0xaa, + 0xc2, 0x3e, 0xaf, 0x60, 0xd7, 0x8c, 0x6b, 0xa5, 0x61, 0x5b, 0x20, 0x51, 0x7f, 0x40, 0xf8, 0x42, + 0xd2, 0x79, 0x68, 0xd6, 0x01, 0x71, 0x56, 0x6c, 0x4e, 0xaa, 0xc2, 0xbe, 0xa0, 0x60, 0xeb, 0xc6, + 0x4a, 0x29, 0x58, 0xde, 0xd3, 0x96, 0xb4, 0xbf, 0x20, 0x7c, 0x51, 0xf7, 0x81, 0x9a, 0x97, 0xf6, + 0xf3, 0x9e, 0x6e, 0x16, 0xab, 0x12, 0xbf, 0xa8, 0x88, 0x37, 0x0c, 0xb3, 0x14, 0xb1, 0x48, 0xd5, + 0x25, 0xf3, 0x03, 0x84, 0xa7, 0x64, 0xb3, 0xa9, 0x71, 0x07, 0xd4, 0xc0, 0x5c, 0x33, 0x5a, 0x95, + 0x74, 0x53, 0x91, 0x9a, 0xc6, 0xb3, 0xe5, 0x6c, 0x2b, 0x58, 0x24, 0x21, 0xbf, 0x42, 0x78, 0x4a, + 0x3e, 0xbf, 0xc3, 0x20, 0x73, 0xcf, 0x73, 0x55, 0xc8, 0x55, 0x05, 0xf9, 0x0c, 0xa5, 0xc3, 0x21, + 0x03, 0x3f, 0x54, 0xf9, 0xd4, 0xc5, 0x63, 0xbd, 0x86, 0x92, 0x0f, 0x7a, 0xdd, 0xb2, 0x5e, 0xd7, + 0x20, 0xd9, 0x6a, 0xda, 0x95, 0xd0, 0x1b, 0x4a, 0x6b, 0x93, 0xac, 0x97, 0x32, 0xc8, 0xfd, 0xa4, + 0x31, 0x39, 0xb6, 0x02, 0xe6, 0xad, 0xa1, 0x97, 0x6f, 0xfe, 0x7a, 0x32, 0x8f, 0x7e, 0x3b, 0x99, + 0x47, 0x7f, 0x9c, 0xcc, 0xa3, 0x77, 0xac, 0xbf, 0xfb, 0x1b, 0xe4, 0xd4, 0x7f, 0x3a, 0x7b, 0xe7, + 0xd5, 0xbf, 0x1a, 0x1b, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0x7c, 0x66, 0x09, 0xdd, 0xf4, 0x11, + 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1937,6 +1955,23 @@ func (m *WorkflowRetryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.NodeFieldSelector) > 0 { + i -= len(m.NodeFieldSelector) + copy(dAtA[i:], m.NodeFieldSelector) + i = encodeVarintWorkflow(dAtA, i, uint64(len(m.NodeFieldSelector))) + i-- + dAtA[i] = 0x22 + } + if m.RestartSuccesful { + i-- + if m.RestartSuccesful { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } if len(m.Namespace) > 0 { i -= len(m.Namespace) copy(dAtA[i:], m.Namespace) @@ -2589,6 +2624,13 @@ func (m *WorkflowRetryRequest) Size() (n int) { if l > 0 { n += 1 + l + sovWorkflow(uint64(l)) } + if m.RestartSuccesful { + n += 2 + } + l = len(m.NodeFieldSelector) + if l > 0 { + n += 1 + l + sovWorkflow(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -3618,6 +3660,58 @@ func (m *WorkflowRetryRequest) Unmarshal(dAtA []byte) error { } m.Namespace = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RestartSuccesful", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowWorkflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RestartSuccesful = bool(v != 0) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeFieldSelector", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowWorkflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthWorkflow + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthWorkflow + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NodeFieldSelector = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipWorkflow(dAtA[iNdEx:]) diff --git a/pkg/apiclient/workflow/workflow.proto b/pkg/apiclient/workflow/workflow.proto index 8f71fd92f0ad..162989128b84 100644 --- a/pkg/apiclient/workflow/workflow.proto +++ b/pkg/apiclient/workflow/workflow.proto @@ -44,6 +44,8 @@ message WorkflowResubmitRequest { message WorkflowRetryRequest { string name = 1; string namespace = 2; + bool restartSuccesful = 3; + string nodeFieldSelector = 4; } message WorkflowResumeRequest { string name = 1; diff --git a/pkg/apiclient/workflow/workflow.swagger.json b/pkg/apiclient/workflow/workflow.swagger.json index b6afdfbdb081..82c78ac50a96 100644 --- a/pkg/apiclient/workflow/workflow.swagger.json +++ b/pkg/apiclient/workflow/workflow.swagger.json @@ -4943,6 +4943,13 @@ }, "namespace": { "type": "string" + }, + "restartSuccesful": { + "type": "boolean", + "format": "boolean" + }, + "nodeFieldSelector": { + "type": "string" } } }, diff --git a/server/workflow/workflow_server.go b/server/workflow/workflow_server.go index 03a72a146428..2e387257939f 100644 --- a/server/workflow/workflow_server.go +++ b/server/workflow/workflow_server.go @@ -220,7 +220,7 @@ func (s *workflowServer) RetryWorkflow(ctx context.Context, req *workflowpkg.Wor return nil, err } - wf, err = util.RetryWorkflow(kubeClient, s.offloadNodeStatusRepo, wfClient.ArgoprojV1alpha1().Workflows(req.Namespace), wf) + wf, err = util.RetryWorkflow(kubeClient, s.offloadNodeStatusRepo, wfClient.ArgoprojV1alpha1().Workflows(req.Namespace), wf, req.RestartSuccesful, req.NodeFieldSelector) if err != nil { return nil, err } diff --git a/test/e2e/cli_test.go b/test/e2e/cli_test.go index 0bc14d30f135..c9334cd48c1e 100644 --- a/test/e2e/cli_test.go +++ b/test/e2e/cli_test.go @@ -476,17 +476,42 @@ func (s *CLISuite) TestWorkflowRetryNoPersistence() { // When this is the case, this behavior is tested in cli_with_server_test.go s.T().SkipNow() } + + var retryTime corev1.Time + s.Given(). - Workflow("@testdata/exit-1.yaml"). + Workflow("@testdata/retry-test.yaml"). When(). SubmitWorkflow(). - WaitForWorkflow(30*time.Second). - Given(). - RunCli([]string{"retry", "exit-1"}, func(t *testing.T, output string, err error) { + WaitForWorkflowToStart(5*time.Second). + WaitForWorkflowCondition(func(wf *wfv1.Workflow) bool { + return wf.Status.AnyActiveSuspendNode() + }, "suspended node", 30*time.Second). + RunCli([]string{"terminate", "retry-test"}, func(t *testing.T, output string, err error) { + if assert.NoError(t, err) { + assert.Contains(t, output, "workflow retry-test terminated") + } + }). + WaitForWorkflowCondition(func(wf *wfv1.Workflow) bool { + retryTime = wf.Status.FinishedAt + return wf.Status.Phase == wfv1.NodeFailed + }, "terminated", 20*time.Second). + RunCli([]string{"retry", "retry-test", "--restart-successful", "--node-field-selector", "templateName==steps-inner"}, func(t *testing.T, output string, err error) { if assert.NoError(t, err) { assert.Contains(t, output, "Name:") assert.Contains(t, output, "Namespace:") } + }). + WaitForWorkflowCondition(func(wf *wfv1.Workflow) bool { + return wf.Status.AnyActiveSuspendNode() + }, "suspended node", 20*time.Second). + Then(). + ExpectWorkflow(func(t *testing.T, _ *corev1.ObjectMeta, status *wfv1.WorkflowStatus) { + outerStepsPodNode := status.Nodes.FindByDisplayName("steps-outer-step1") + innerStepsPodNode := status.Nodes.FindByDisplayName("steps-inner-step1") + + assert.True(t, outerStepsPodNode.FinishedAt.Before(&retryTime)) + assert.True(t, retryTime.Before(&innerStepsPodNode.FinishedAt)) }) } diff --git a/test/e2e/testdata/retry-test.yaml b/test/e2e/testdata/retry-test.yaml new file mode 100644 index 000000000000..88fd652111c4 --- /dev/null +++ b/test/e2e/testdata/retry-test.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + name: retry-test + labels: + argo-e2e: "true" +spec: + entrypoint: steps-outer + templates: + - name: steps-outer + steps: + - - name: steps-outer-step1 + template: whalesay + - - name: steps-outer-step2 + template: steps-inner + + - name: steps-inner + steps: + - - name: steps-inner-step1 + template: whalesay + - - name: steps-inner-step2 + template: approve + + - name: approve + suspend: {} + + - name: whalesay + container: + image: docker/whalesay:latest + command: [cowsay] + args: ["hello world"] \ No newline at end of file diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go index 5fe538b4f35e..76d1e65bf8a8 100644 --- a/workflow/controller/operator.go +++ b/workflow/controller/operator.go @@ -218,7 +218,7 @@ func (woc *wfOperationCtx) operate() { woc.workflowDeadline = woc.getWorkflowDeadline() err := woc.podReconciliation() if err == nil { - err = woc.failSuspendedNodesAfterDeadline() + err = woc.failSuspendedNodesAfterDeadlineOrShutdown() } if err != nil { woc.log.Errorf("%s error: %+v", woc.wf.ObjectMeta.Name, err) @@ -636,6 +636,17 @@ func (woc *wfOperationCtx) processNodeRetries(node *wfv1.NodeStatus, retryStrate return woc.markNodePhase(node.Name, wfv1.NodeSucceeded), true, nil } + if woc.wf.Spec.Shutdown != "" || (woc.workflowDeadline != nil && time.Now().UTC().After(*woc.workflowDeadline)) { + var message string + if woc.wf.Spec.Shutdown != "" { + message = fmt.Sprintf("Stopped with strategy '%s'", woc.wf.Spec.Shutdown) + } else { + message = fmt.Sprintf("retry exceeded workflow deadline %s", *woc.workflowDeadline) + } + woc.log.Infoln(message) + return woc.markNodePhase(node.Name, lastChildNode.Phase, message), true, nil + } + if retryStrategy.Backoff != nil { // Process max duration limit if retryStrategy.Backoff.MaxDuration != "" && len(node.Children) > 0 { @@ -808,11 +819,17 @@ func (woc *wfOperationCtx) shouldPrintPodSpec(node wfv1.NodeStatus) bool { } //fails any suspended nodes if the workflow deadline has passed -func (woc *wfOperationCtx) failSuspendedNodesAfterDeadline() error { - if woc.workflowDeadline != nil && time.Now().UTC().After(*woc.workflowDeadline) { +func (woc *wfOperationCtx) failSuspendedNodesAfterDeadlineOrShutdown() error { + if woc.wf.Spec.Shutdown != "" || (woc.workflowDeadline != nil && time.Now().UTC().After(*woc.workflowDeadline)) { for _, node := range woc.wf.Status.Nodes { if node.IsActiveSuspendNode() { - woc.markNodePhase(node.Name, wfv1.NodeFailed, fmt.Sprintf("step exceeded workflow deadline %s", *woc.workflowDeadline)) + var message string + if woc.wf.Spec.Shutdown != "" { + message = fmt.Sprintf("Stopped with strategy '%s'", woc.wf.Spec.Shutdown) + } else { + message = fmt.Sprintf("step exceeded workflow deadline %s", *woc.workflowDeadline) + } + woc.markNodePhase(node.Name, wfv1.NodeFailed, message) } } } diff --git a/workflow/util/util.go b/workflow/util/util.go index 4528988f9bc3..f0f72f4cb7fd 100644 --- a/workflow/util/util.go +++ b/workflow/util/util.go @@ -421,6 +421,20 @@ func ResumeWorkflow(wfIf v1alpha1.WorkflowInterface, repo sqldb.OffloadNodeStatu } } +func selectorMatchesNode(selector fields.Selector, node wfv1.NodeStatus) bool { + nodeFields := fields.Set{ + "displayName": node.DisplayName, + "templateName": node.TemplateName, + } + if node.Inputs != nil { + for _, inParam := range node.Inputs.Parameters { + nodeFields[fmt.Sprintf("inputs.parameters.%s.value", inParam.Name)] = *inParam.Value + } + } + + return selector.Matches(nodeFields) +} + func updateWorkflowNodeByKey(wfIf v1alpha1.WorkflowInterface, workflowName string, nodeFieldSelector string, phase wfv1.NodePhase, message string) error { selector, err := fields.ParseSelector(nodeFieldSelector) @@ -441,16 +455,7 @@ func updateWorkflowNodeByKey(wfIf v1alpha1.WorkflowInterface, workflowName strin nodeUpdated := false for nodeID, node := range wf.Status.Nodes { if node.IsActiveSuspendNode() { - nodeFields := fields.Set{ - "displayName": node.DisplayName, - } - if node.Inputs != nil { - for _, inParam := range node.Inputs.Parameters { - nodeFields[fmt.Sprintf("inputs.parameters.%s.value", inParam.Name)] = *inParam.Value - } - } - - if selector.Matches(nodeFields) { + if selectorMatchesNode(selector, node) { node.Phase = phase node.FinishedAt = metav1.Time{Time: time.Now().UTC()} if len(message) > 0 { @@ -612,7 +617,7 @@ func convertNodeID(newWf *wfv1.Workflow, regex *regexp.Regexp, oldNodeID string, } // RetryWorkflow updates a workflow, deleting all failed steps as well as the onExit node (and children) -func RetryWorkflow(kubeClient kubernetes.Interface, repo sqldb.OffloadNodeStatusRepo, wfClient v1alpha1.WorkflowInterface, wf *wfv1.Workflow) (*wfv1.Workflow, error) { +func RetryWorkflow(kubeClient kubernetes.Interface, repo sqldb.OffloadNodeStatusRepo, wfClient v1alpha1.WorkflowInterface, wf *wfv1.Workflow, restartSuccessful bool, nodeFieldSelector string) (*wfv1.Workflow, error) { switch wf.Status.Phase { case wfv1.NodeFailed, wfv1.NodeError: default: @@ -634,12 +639,12 @@ func RetryWorkflow(kubeClient kubernetes.Interface, repo sqldb.OffloadNodeStatus newWF.Status.Phase = wfv1.NodeRunning newWF.Status.Message = "" newWF.Status.FinishedAt = metav1.Time{} + newWF.Spec.Shutdown = "" if newWF.Spec.ActiveDeadlineSeconds != nil && *newWF.Spec.ActiveDeadlineSeconds == 0 { // if it was terminated, unset the deadline newWF.Spec.ActiveDeadlineSeconds = nil } - // Iterate the previous nodes. If it was successful Pod carry it forward newNodes := make(map[string]wfv1.NodeStatus) onExitNodeName := wf.ObjectMeta.Name + ".onExit" nodes := wf.Status.Nodes @@ -654,10 +659,22 @@ func RetryWorkflow(kubeClient kubernetes.Interface, repo sqldb.OffloadNodeStatus } } + // Get all children of nodes that match filter + nodeIDsToReset, err := getNodeIDsToReset(restartSuccessful, nodeFieldSelector, nodes) + if err != nil { + return nil, err + } + + // Iterate the previous nodes. If it was successful Pod carry it forward for _, node := range nodes { + doForceResetNode := false + if _, present := nodeIDsToReset[node.ID]; present { + // if we are resetting this node then don't carry it across regardless of its phase + doForceResetNode = true + } switch node.Phase { case wfv1.NodeSucceeded, wfv1.NodeSkipped: - if !strings.HasPrefix(node.Name, onExitNodeName) { + if !strings.HasPrefix(node.Name, onExitNodeName) && !doForceResetNode { newNodes[node.ID] = node continue } @@ -719,6 +736,37 @@ func RetryWorkflow(kubeClient kubernetes.Interface, repo sqldb.OffloadNodeStatus return wfClient.Update(newWF) } +func getNodeIDsToReset(restartSuccessful bool, nodeFieldSelector string, nodes wfv1.Nodes) (map[string]bool, error) { + nodeIDsToReset := make(map[string]bool) + if !restartSuccessful || len(nodeFieldSelector) == 0 { + return nodeIDsToReset, nil + } + + selector, err := fields.ParseSelector(nodeFieldSelector) + if err != nil { + return nil, err + } else { + for _, node := range nodes { + if selectorMatchesNode(selector, node) { + //traverse all children of the node + var queue []string + queue = append(queue, node.ID) + + for len(queue) > 0 { + childNode := queue[0] + //if the child isn't already in nodeIDsToReset then we add it and traverse its children + if _, present := nodeIDsToReset[childNode]; !present { + nodeIDsToReset[childNode] = true + queue = append(queue, nodes[childNode].Children...) + } + queue = queue[1:] + } + } + } + } + return nodeIDsToReset, nil +} + var errSuspendedCompletedWorkflow = errors.Errorf(errors.CodeBadRequest, "cannot suspend completed workflows") // IsWorkflowSuspended returns whether or not a workflow is considered suspended diff --git a/workflow/util/util_test.go b/workflow/util/util_test.go index 0f28c6ab0106..a5c49b6cb242 100644 --- a/workflow/util/util_test.go +++ b/workflow/util/util_test.go @@ -456,7 +456,7 @@ func TestRetryWorkflowCompressed(t *testing.T) { clearFunc = packer.SetMaxWorkflowSize(1557) defer clearFunc() - wf, err := RetryWorkflow(kubeCs, sqldb.ExplosiveOffloadNodeStatusRepo, wfIf, origWf) + wf, err := RetryWorkflow(kubeCs, sqldb.ExplosiveOffloadNodeStatusRepo, wfIf, origWf, false, "") assert.NoError(t, err) assert.NotEmpty(t, wf.Status.CompressedNodes) } @@ -481,7 +481,7 @@ func TestRetryWorkflowOffloaded(t *testing.T) { offloadNodeStatusRepo.On("Get", "7e74dbb9-d681-4c22-9bed-a581ec28383f", "123").Return(origNodes, nil) offloadNodeStatusRepo.On("Save", "7e74dbb9-d681-4c22-9bed-a581ec28383f", mock.Anything, mock.Anything).Return("1234", nil) - _, err = RetryWorkflow(kubeCs, offloadNodeStatusRepo, wfIf, origWf) + _, err = RetryWorkflow(kubeCs, offloadNodeStatusRepo, wfIf, origWf, false, "") assert.NoError(t, err) wf, err := wfIf.Get("fail-template", metav1.GetOptions{})