test: Fix teething problems (argoproj#2630)

go.mod

@@ -80,7 +80,7 @@ require (
  golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
  golang.org/x/net v0.0.0-20200301022130-244492dfa37a
  golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
- golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae // indirect
+ golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a // indirect
  google.golang.org/api v0.20.0
  google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24
  google.golang.org/grpc v1.28.0

go.sum

@@ -618,8 +618,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae h1:nMRC2i7oRGagyN5g3chxBF7gb2do/tIJ3ihoEnvYvsU=
-golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a h1:3CGHZdxu1QXyxpZvKA3QHjIyj6ia66IIHQ3O0lKHgyQ=
+golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

test/e2e/cli_test.go

@@ -228,7 +228,7 @@ func (s *CLISuite) TestRoot() {
  s.Given().CronWorkflow("@testdata/basic.yaml").
  When().
  CreateCronWorkflow().
- RunCli([]string{"submit", "--from", "cronwf/test-cron-wf-basic"}, func(t *testing.T, output string, err error) {
+ RunCli([]string{"submit", "--from", "cronwf/test-cron-wf-basic", "-l", "argo-e2e=true"}, func(t *testing.T, output string, err error) {
  assert.NoError(t, err)
  assert.Contains(t, output, "Name: test-cron-wf-basic-")
  r := regexp.MustCompile(`Name:\s+?(test-cron-wf-basic-[a-z0-9]+)`)
@@ -238,7 +238,7 @@ func (s *CLISuite) TestRoot() {
  }
  createdWorkflowName = res[1]
  }).
- WaitForWorkflowName(createdWorkflowName, 15*time.Second).
+ WaitForWorkflowName(createdWorkflowName, 20*time.Second).
  Then().
  ExpectWorkflowName(createdWorkflowName, func(t *testing.T, metadata *corev1.ObjectMeta, status *wfv1.WorkflowStatus) {
  assert.Equal(t, wfv1.NodeSucceeded, status.Phase)
@@ -320,7 +320,7 @@ func (s *CLISuite) TestWorkflowDelete() {
  Workflow("@smoke/basic.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow(15*time.Second).
+ WaitForWorkflow(20*time.Second).
  Given().
  RunCli([]string{"delete", "basic"}, func(t *testing.T, output string, err error) {
  if assert.NoError(t, err) {
@@ -333,7 +333,7 @@ func (s *CLISuite) TestWorkflowDelete() {
  Workflow("@smoke/basic.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow(15*time.Second).
+ WaitForWorkflow(20*time.Second).
  Given().
  RunCli([]string{"delete", "--all", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) {
  if assert.NoError(t, err) {
@@ -354,7 +354,7 @@ func (s *CLISuite) TestWorkflowDelete() {
  }
  }).
  When().
- WaitForWorkflow(15*time.Second).
+ WaitForWorkflow(20*time.Second).
  Given().
  RunCli([]string{"delete", "--completed", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) {
  if assert.NoError(t, err) {
@@ -367,7 +367,7 @@ func (s *CLISuite) TestWorkflowDelete() {
  Workflow("@smoke/basic.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow(15*time.Second).
+ WaitForWorkflow(20*time.Second).
  Given().
  RunCli([]string{"delete", "--older", "1d", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) {
  if assert.NoError(t, err) {
@@ -559,7 +559,7 @@ func (s *CLISuite) TestWorkflowResubmit() {
  Workflow("@testdata/exit-1.yaml").
  When().
  SubmitWorkflow().
- WaitForWorkflow(15*time.Second).
+ WaitForWorkflow(20*time.Second).
  Given().
  RunCli([]string{"resubmit", "--memoized", "exit-1"}, func(t *testing.T, output string, err error) {
  if assert.NoError(t, err) {

test/e2e/fixtures/e2e_suite.go

@@ -29,21 +29,24 @@ const Label = "argo-e2e"
 // Cron tests run in parallel, so use a different label so they are not deleted when a new test runs
 const LabelCron = Label + "-cron"
 
-var gitBranch string
+var imageTag string
 var k3d bool
 
 func init() {
  output, err := runCli("git", "rev-parse", "--abbrev-ref=loose", "HEAD")
  if err != nil {
  panic(err)
  }
- gitBranch = strings.TrimSpace(output)
+ imageTag = strings.TrimSpace(output)
+ if imageTag == "master" {
+ imageTag = "latest"
+ }
  context, err := runCli("kubectl", "config", "current-context")
  if err != nil {
  panic(err)
  }
  k3d = strings.TrimSpace(context) == "k3s-default"
- log.WithFields(log.Fields{"gitBranch": gitBranch, "k3d": k3d}).Info()
+ log.WithFields(log.Fields{"imageTag": imageTag, "k3d": k3d}).Info()
 }
 
 type E2ESuite struct {
@@ -62,7 +65,7 @@ type E2ESuite struct {
  images map[string]bool
  // Guard-rail.
  // The number of archived workflows. If is changes between two tests, we have a problem.
- numArchivedWorkflows int
+ numWorkflows int
 }
 
 func (s *E2ESuite) SetupSuite() {
@@ -113,28 +116,24 @@ func (s *E2ESuite) BeforeTest(suiteName, testName string) {
  s.DeleteResources(Label)
  s.images = s.listImages()
  s.importImages()
- numArchivedWorkflows := s.countArchivedWorkflows()
- if s.numArchivedWorkflows > 0 && s.numArchivedWorkflows != numArchivedWorkflows {
- s.T().Fatal("there should almost never be a change to the number of archived workflows between tests, this means the last test (no the current test) is bad and needs fixing - note this guard-rail does not work across test suites")
+ numWorkflows := s.countWorkflows()
+ if s.numWorkflows > 0 && s.numWorkflows != numWorkflows {
+ s.T().Fatal("there should almost never be a change to the number of workflows between tests, this means the last test (not the current test) is bad and needs fixing - note this guard-rail does not work across test suites")
  }
- s.numArchivedWorkflows = numArchivedWorkflows
+ s.numWorkflows = numWorkflows
 }
 
-func (s *E2ESuite) countArchivedWorkflows() int {
- if s.Persistence.IsEnabled() {
- workflows, err := s.Persistence.workflowArchive.ListWorkflows(Namespace, time.Time{}, time.Time{}, nil, 0, 0)
- s.CheckError(err)
- return len(workflows)
- }
- return 0
+func (s *E2ESuite) countWorkflows() int {
+ workflows, err := s.wfClient.List(metav1.ListOptions{})
+ s.CheckError(err)
+ return len(workflows.Items)
 }
 
 func (s *E2ESuite) importImages() {
  // If we are running K3D we should re-import these prior to running tests, as they may have been evicted.
  if k3d {
- images := s.listImages()
- for _, n := range []string{"docker.io/argoproj/argoexec:" + gitBranch, "docker.io/library/cowsay:v1"} {
- if !images[n] {
+ for _, n := range []string{"docker.io/argoproj/argoexec:" + imageTag, "docker.io/library/cowsay:v1"} {
+ if !s.images[n] {
  _, err := runCli("k3d", "import-images", n)
  s.CheckError(err)
  }
@@ -326,11 +325,9 @@ func (s *E2ESuite) AfterTest(_, _ string) {
  // Using an arbitrary image will result in slow and flakey tests as we can't really predict when they'll be
  // downloaded or evicted. To keep tests fast and reliable you must use whitelisted images.
  imageWhitelist := map[string]bool{
- "docker.io/argoproj/argoexec:" + gitBranch: true,
- "docker.io/library/cowsay:v1": true,
- "docker.io/library/python:alpine3.6": true,
- // why this different name?
- "python:alpine3.6": true,
+ "docker.io/argoproj/argoexec:" + imageTag: true,
+ "docker.io/library/cowsay:v1": true,
+ "docker.io/library/python:alpine3.6": true,
  }
  for n := range s.listImages() {
  if !s.images[n] && !imageWhitelist[n] {