ci: Split e2e tests to make them faster (argoproj#4404)

Signed-off-by: Alex Collins <[email protected]>

* ok

Signed-off-by: Alex Collins <[email protected]>

.github/workflows/ci-build.yaml

@@ -33,7 +33,7 @@ jobs:
  - name: Run tests
  env:
  GOPATH: /home/runner/go
- run: make test
+ run: make test STATIC_FILES=false
 
  e2e-tests:
  name: E2E Tests
@@ -43,14 +43,18 @@ jobs:
  fail-fast: false
  matrix:
  # kubelet is not included because it'd take ages to get it working methinks
- test: [ "smoke", "test-e2e", "test-e2e-cron" ]
+ test: [ "smoke", "test-e2e", "test-cli", "test-e2e-cron" ]
  containerRuntimeExecutor: [ "docker", "k8sapi", "pns" ]
  # ok, so we're only running `smoke` for all CREs,
  exclude:
  - test: test-e2e
  containerRuntimeExecutor: k8sapi
  - test: test-e2e
  containerRuntimeExecutor: pns
+ - test: test-cli
+ containerRuntimeExecutor: k8sapi
+ - test: test-cli
+ containerRuntimeExecutor: pns
  - test: test-e2e-cron
  containerRuntimeExecutor: k8sapi
  - test: test-e2e-cron
@@ -97,14 +101,16 @@ jobs:
  mkdir -p /tmp/log/argo-e2e
  git fetch --tags
  KUBECONFIG=~/.kube/config make start PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
+ - name: Install gotestsum
+ run: go install gotest.tools/gotestsum
  - name: Wait for Argo Server to be ready
  env:
  GOPATH: /home/runner/go
  run: make wait
  - name: Run tests
  env:
  GOPATH: /home/runner/go
- run: make ${{ matrix.test }}
+ run: make ${{ matrix.test }} GOTEST='gotestsum --format testname --'
  - name: Upload logs
  if: ${{ always() }}
  uses: actions/upload-artifact@v1

.golangci.yml

@@ -8,6 +8,7 @@ run:
  - server/static/files.go
  build-tags:
  - e2e
+ - cli
 linters:
  enable:
  - goimports

Makefile

@@ -48,6 +48,7 @@ CONTROLLER_IMAGE_FILE := dist/controller-image.marker
 # perform static compilation
 STATIC_BUILD ?= true
 STATIC_FILES ?= true
+GOTEST ?= go test
 PROFILE ?= minimal
 # whether or not to start the Argo Service in TLS mode
 SECURE := false
@@ -379,7 +380,7 @@ endif
 # for local we have a faster target that prints to stdout, does not use json, and can cache because it has no coverage
 .PHONY: test
 test: server/static/files.go
- env KUBECONFIG=/dev/null go test ./...
+ env KUBECONFIG=/dev/null $(GOTEST) ./...
 
 dist/$(PROFILE).yaml: $(MANIFESTS) $(E2E_MANIFESTS) /usr/local/bin/kustomize
  mkdir -p dist
@@ -457,15 +458,19 @@ mysql-cli:
 
 .PHONY: test-e2e
 test-e2e:
- go test -timeout 20m -count 1 --tags e2e -p 1 --short ./test/e2e
+ $(GOTEST) -timeout 10m -count 1 --tags e2e -p 1 --short ./test/e2e
+
+.PHONY: test-cli
+test-cli:
+ $(GOTEST) -timeout 10m -count 1 --tags cli -p 1 --short ./test/e2e
 
 .PHONY: test-e2e-cron
 test-e2e-cron:
- go test -count 1 --tags e2e -parallel 10 -run CronSuite ./test/e2e
+ $(GOTEST) -count 1 --tags e2e -parallel 10 -run CronSuite ./test/e2e
 
 .PHONY: smoke
 smoke:
- go test -count 1 --tags e2e -p 1 -run SmokeSuite ./test/e2e
+ $(GOTEST) -count 1 --tags e2e -p 1 -run SmokeSuite ./test/e2e
 
 # clean
 

test/e2e/argo_server_test.go

@@ -1,10 +1,9 @@
-// +build e2e
+// +build cli
 
 package e2e
 
 import (
  "bufio"
- "crypto/tls"
  "io/ioutil"
  "net/http"
  "strings"
@@ -28,13 +27,6 @@ import (
 
 const baseUrl = "http:https://localhost:2746"
 
-var httpClient = &http.Client{
- Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- return http.ErrUseLastResponse
- },
-}
-
 // ensure basic HTTP functionality works,
 // testing behaviour really is a non-goal
 type ArgoServerSuite struct {
@@ -49,13 +41,6 @@ func (s *ArgoServerSuite) BeforeTest(suiteName, testName string) {
  s.CheckError(err)
 }
 
-type httpLogger struct {
-}
-
-func (d *httpLogger) Logf(fmt string, args ...interface{}) {
- log.Debugf(fmt, args...)
-}
-
 func (s *ArgoServerSuite) e() *httpexpect.Expect {
  return httpexpect.
  WithConfig(httpexpect.Config{

test/e2e/cli_test.go

@@ -1,4 +1,4 @@
-// +build e2e
+// +build cli
 
 package e2e
 

test/e2e/cli_with_server_test.go

@@ -1,4 +1,4 @@
-// +build e2e
+// +build cli
 
 package e2e
 

test/e2e/client.go

@@ -0,0 +1,13 @@
+package e2e
+
+import (
+ "crypto/tls"
+ "net/http"
+)
+
+var httpClient = &http.Client{
+ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+}

test/e2e/http_logger.go

@@ -0,0 +1,10 @@
+package e2e
+
+import log "github.com/sirupsen/logrus"
+
+type httpLogger struct {
+}
+
+func (d *httpLogger) Logf(fmt string, args ...interface{}) {
+ log.Debugf(fmt, args...)
+}