test: Make tests more robust, again (argoproj#5346)

test/e2e/argo_server_test.go

@@ -495,9 +495,7 @@ func (s *ArgoServerSuite) TestPermission() {
  "name": "run-workflow",
  "container": {
  "image": "argoproj/argosay:v2",
- "imagePullPolicy": "IfNotPresent",
- "command": ["sh"],
- "args": ["-c", "sleep 1"]
+ "args": [ "sleep", "1" ]
  }
  }
  ],

test/e2e/artifacts_test.go

@@ -23,22 +23,15 @@ func (s *ArtifactsSuite) TestInputOnMount() {
  When().
  SubmitWorkflow().
  WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ArtifactsSuite) TestOutputOnMount() {
  s.Given().
  Workflow("@testdata/output-on-mount-workflow.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ArtifactsSuite) TestOutputOnInput() {
@@ -47,11 +40,7 @@ func (s *ArtifactsSuite) TestOutputOnInput() {
  Workflow("@testdata/output-on-input-workflow.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ArtifactsSuite) TestArtifactPassing() {
@@ -60,11 +49,7 @@ func (s *ArtifactsSuite) TestArtifactPassing() {
  Workflow("@smoke/artifact-passing.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ArtifactsSuite) TestDefaultParameterOutputs() {
@@ -105,10 +90,9 @@ spec:
 `).
  When().
  SubmitWorkflow().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeSucceeded).
  Then().
  ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
  assert.True(t, status.Nodes.Any(func(node wfv1.NodeStatus) bool {
  if node.Outputs != nil {
  for _, param := range node.Outputs.Parameters {
@@ -128,11 +112,7 @@ func (s *ArtifactsSuite) TestSameInputOutputPathOptionalArtifact() {
  Workflow("@testdata/same-input-output-path-optional.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ArtifactsSuite) TestOutputArtifactS3BucketCreationEnabled() {
@@ -141,22 +121,17 @@ func (s *ArtifactsSuite) TestOutputArtifactS3BucketCreationEnabled() {
  Workflow("@testdata/output-artifact-with-s3-bucket-creation-enabled.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ArtifactsSuite) TestOutputResult() {
  s.Given().
  Workflow("@testdata/output-result-workflow.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeSucceeded).
  Then().
  ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
  n := status.Nodes.FindByDisplayName("a")
  if assert.NotNil(t, n) {
  assert.NotNil(t, n.Outputs.ExitCode)

test/e2e/cli_test.go

@@ -449,11 +449,7 @@ func (s *CLISuite) TestWorkflowSuspendResume() {
  assert.Contains(t, output, "workflow sleep-3s resumed")
  }
  }).
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *CLISuite) TestNodeSuspendResume() {
@@ -478,9 +474,7 @@ func (s *CLISuite) TestNodeSuspendResume() {
  assert.Contains(t, output, "workflow node-suspend stopped")
  }
  }).
- WaitForWorkflow(fixtures.Condition(func(wf *wfv1.Workflow) (bool, string) {
- return wf.Status.Phase == wfv1.WorkflowFailed, "suspended node"
- })).
+ WaitForWorkflow(fixtures.ToBeFailed).
  Then().
  ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
  if assert.Equal(t, wfv1.WorkflowFailed, status.Phase) {
@@ -1186,52 +1180,6 @@ func (s *CLISuite) TestWorkflowTemplateRefSubmit() {
  })
 }
 
-func (s *CLISuite) TestWorkflowLevelSemaphore() {
- semaphoreData := map[string]string{
- "workflow": "1",
- }
- s.Need(Offloading)
- s.Given().
- Workflow("@testdata/semaphore-wf-level.yaml").
- When().
- CreateConfigMap("my-config", semaphoreData).
- RunCli([]string{"submit", "testdata/semaphore-wf-level-1.yaml", "-l", "workflows.argoproj.io/test=true"}, func(t *testing.T, output string, err error) {
- if assert.NoError(t, err) {
- assert.Contains(t, output, "semaphore-wf-level-1")
- }
- }).
- SubmitWorkflow().
- WaitForWorkflow(fixtures.Condition(func(wf *wfv1.Workflow) (bool, string) {
- return wf.Status.Phase == wfv1.WorkflowUnknown, "Workflow is waiting for lock"
- })).
- WaitForWorkflow().
- DeleteConfigMap("my-config").
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
-}
-
-func (s *CLISuite) TestTemplateLevelSemaphore() {
- semaphoreData := map[string]string{
- "template": "1",
- }
-
- s.Need(Offloading)
- s.Given().
- Workflow("@testdata/semaphore-tmpl-level.yaml").
- When().
- CreateConfigMap("my-config", semaphoreData).
- SubmitWorkflow().
- WaitForWorkflow(fixtures.Condition(func(wf *wfv1.Workflow) (bool, string) {
- return wf.Status.Phase == wfv1.WorkflowRunning, "waiting for Workflow to run"
- }), 10*time.Second).
- RunCli([]string{"get", "semaphore-tmpl-level"}, func(t *testing.T, output string, err error) {
- assert.Contains(t, output, "Waiting for")
- }).
- WaitForWorkflow()
-}
-
 func (s *CLISuite) TestRetryOmit() {
  s.Need(Offloading)
  s.Given().

test/e2e/cluster_workflow_template_test.go

@@ -7,9 +7,7 @@ import (
 
  "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/suite"
- v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
- "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
  "github.com/argoproj/argo-workflows/v3/test/e2e/fixtures"
 )
 
@@ -26,11 +24,7 @@ func (s *ClusterWorkflowTemplateSuite) TestSubmitClusterWorkflowTemplate() {
  RunCli([]string{"submit", "--from", "clusterworkflowtemplate/cluster-workflow-template-whalesay-template", "--name", "my-wf", "-l", "workflows.argoproj.io/test=true"}, func(t *testing.T, output string, err error) {
  assert.NoError(t, err)
  }).
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, metadata *v1.ObjectMeta, status *v1alpha1.WorkflowStatus) {
- assert.Equal(t, status.Phase, v1alpha1.WorkflowSucceeded)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func (s *ClusterWorkflowTemplateSuite) TestNestedClusterWorkflowTemplate() {
@@ -63,11 +57,7 @@ spec:
  value: hello from nested
 `).When().
  SubmitWorkflow().
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, metadata *v1.ObjectMeta, status *v1alpha1.WorkflowStatus) {
- assert.Equal(t, v1alpha1.WorkflowSucceeded, status.Phase)
- })
+ WaitForWorkflow(fixtures.ToBeSucceeded)
 }
 
 func TestClusterWorkflowTemplateSuite(t *testing.T) {

test/e2e/estimated_duration_test.go

@@ -25,10 +25,9 @@ func (s *EstimatedDurationSuite) TestWorkflowTemplate() {
  SubmitWorkflowsFromWorkflowTemplates().
  WaitForWorkflow().
  SubmitWorkflowsFromWorkflowTemplates().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeSucceeded).
  Then().
  ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
  assert.NotEmpty(t, status.EstimatedDuration)
  assert.NotEmpty(t, status.Nodes[metadata.Name].EstimatedDuration)
  })
@@ -42,10 +41,9 @@ func (s *EstimatedDurationSuite) TestClusterWorkflowTemplate() {
  SubmitWorkflowsFromClusterWorkflowTemplates().
  WaitForWorkflow().
  SubmitWorkflowsFromClusterWorkflowTemplates().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeSucceeded).
  Then().
  ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
  assert.NotEmpty(t, status.EstimatedDuration)
  assert.NotEmpty(t, status.Nodes[metadata.Name].EstimatedDuration)
  })
@@ -59,10 +57,9 @@ func (s *EstimatedDurationSuite) TestCronWorkflow() {
  SubmitWorkflowsFromCronWorkflows().
  WaitForWorkflow().
  SubmitWorkflowsFromCronWorkflows().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeSucceeded).
  Then().
  ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
  assert.NotEmpty(t, status.EstimatedDuration)
  assert.NotEmpty(t, status.Nodes[metadata.Name].EstimatedDuration)
  })

test/e2e/fixtures/when.go

@@ -166,6 +166,10 @@ func (w *When) CreateCronWorkflow() *When {
 type Condition func(wf *wfv1.Workflow) (bool, string)
 
 var (
+ ToBeRunning = ToHavePhase(wfv1.WorkflowRunning)
+ ToBeSucceeded = ToHavePhase(wfv1.WorkflowSucceeded)
+ ToBeErrored = ToHavePhase(wfv1.WorkflowError)
+ ToBeFailed = ToHavePhase(wfv1.WorkflowFailed)
  ToBeCompleted Condition = func(wf *wfv1.Workflow) (bool, string) {
  return wf.Labels[common.LabelKeyCompleted] == "true", "to be completed"
  }
@@ -177,10 +181,6 @@ var (
  }
 )
 
-var ToBeSucceeded Condition = func(wf *wfv1.Workflow) (bool, string) {
- return wf.Status.Phase == wfv1.WorkflowSucceeded, "to be succeeded"
-}
-
 // `ToBeDone` replaces `ToFinish` which also makes sure the workflow is both complete not pending archiving.
 // This additional check is not needed for most use case, however in `AfterTest` we delete the workflow and this
 // creates a lot of warning messages in the logs that are cause by misuse rather than actual problems.
@@ -193,6 +193,12 @@ var ToBeArchived Condition = func(wf *wfv1.Workflow) (bool, string) {
  return wf.Labels[common.LabelKeyWorkflowArchivingStatus] == "Archived", "to be archived"
 }
 
+var ToHavePhase = func(p wfv1.WorkflowPhase) Condition {
+ return func(wf *wfv1.Workflow) (bool, string) {
+ return wf.Status.Phase == p, fmt.Sprintf("to be %s", p)
+ }
+}
+
 var ToBeWaitingOnAMutex Condition = func(wf *wfv1.Workflow) (bool, string) {
  return wf.Status.Synchronization != nil && wf.Status.Synchronization.Mutex != nil, "to be waiting on a mutex"
 }

test/e2e/functional_test.go

@@ -60,36 +60,6 @@ func (s *FunctionalSuite) TestWorkflowLevelErrorRetryPolicy() {
  })
 }
 
-func (s *FunctionalSuite) TestSynchronizationWfLevelMutex() {
- s.Given().
- Workflow("@functional/synchronization-mutex-wf-level-1.yaml").
- When().
- SubmitWorkflow().
- Given().
- Workflow("@functional/synchronization-mutex-wf-level.yaml").
- When().
- SubmitWorkflow().
- WaitForWorkflow(fixtures.ToBeWaitingOnAMutex).
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
-}
-
-func (s *FunctionalSuite) TestTemplateLevelMutex() {
- s.Given().
- Workflow("@functional/synchronization-mutex-tmpl-level.yaml").
- When().
- SubmitWorkflow().
- WaitForWorkflow(fixtures.ToBeWaitingOnAMutex).
- WaitForWorkflow().
- Then().
- ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
- })
-}
-
 func (s *FunctionalSuite) TestWorkflowTTL() {
  s.Given().
  Workflow(`

test/e2e/invalid_command_test.go

@@ -0,0 +1,34 @@
+// +build executor
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/suite"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+ "github.com/argoproj/argo-workflows/v3/test/e2e/fixtures"
+)
+
+type InvalidCommandSuite struct {
+ fixtures.E2ESuite
+}
+
+func (s *InvalidCommandSuite) TestInvalidCommand() {
+ s.Given().
+ Workflow("@testdata/cannot-start-workflow.yaml").
+ When().
+ SubmitWorkflow().
+ WaitForWorkflow(fixtures.ToBeFailed).
+ Then().
+ ExpectWorkflow(func(t *testing.T, _ *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
+ assert.Contains(t, status.Message, "executable file not found")
+ })
+}
+
+func TestInvalidCommandSuite(t *testing.T) {
+ suite.Run(t, new(InvalidCommandSuite))
+}

test/e2e/pod_cleanup_test.go

@@ -70,10 +70,9 @@ spec:
 `).
  When().
  SubmitWorkflow().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeFailed).
  Then().
  ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowFailed, status.Phase)
  assert.True(t, strings.Contains(status.Message, "failed to parse label selector"))
  })
 }

test/e2e/progress_test.go

@@ -22,10 +22,9 @@ func (s *ProgressSuite) TestDefaultProgress() {
  Workflow("@testdata/basic-workflow.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow().
+ WaitForWorkflow(fixtures.ToBeSucceeded).
  Then().
  ExpectWorkflow(func(t *testing.T, metadata *metav1.ObjectMeta, status *wfv1.WorkflowStatus) {
- assert.Equal(t, wfv1.WorkflowSucceeded, status.Phase)
  assert.Equal(t, wfv1.Progress("1/1"), status.Progress)
  assert.Equal(t, wfv1.Progress("1/1"), status.Nodes[metadata.Name].Progress)
  })