test: E2E test refactoring (argoproj#3849)

.github/workflows/ci-build.yaml

@@ -56,7 +56,6 @@ jobs:
  env:
  GOPATH: /home/runner/go
  PROFILE: mysql
- STATIC_FILES: false
  run: |
  echo '127.0.0.1 dex' | sudo tee -a /etc/hosts
  echo '127.0.0.1 minio' | sudo tee -a /etc/hosts
@@ -65,8 +64,8 @@ jobs:
  mkdir -p dist
  mkdir -p /tmp/log/argo-e2e
  git fetch --tags
- KUBECONFIG=~/.kube/config make status install controller cli executor-image PROFILE=$PROFILE E2E_EXECUTOR=docker DEV_IMAGE=true
- KUBECONFIG=~/.kube/config make start PROFILE=$PROFILE E2E_EXECUTOR=docker DEV_IMAGE=true 2>&1 > /tmp/log/argo-e2e/argo.log &
+ KUBECONFIG=~/.kube/config make status install controller cli executor-image PROFILE=$PROFILE E2E_EXECUTOR=docker DEV_IMAGE=true STATIC_FILES=false
+ KUBECONFIG=~/.kube/config make start PROFILE=$PROFILE E2E_EXECUTOR=docker DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
  - name: Wait for Argo Server to be ready
  if: ${{ matrix.test != 'test' }}
  env:
@@ -75,24 +74,13 @@ jobs:
  - name: Run tests
  env:
  GOPATH: /home/runner/go
- run: |
- trap 'make test-results/junit.xml' EXIT
- make ${{ matrix.test }}
- - name: Print test error annotations
- if: failure()
- run: make test-report
- - name: Upload test results
- if: failure()
- uses: actions/upload-artifact@v1
- with:
- name: ${{ matrix.test }}
- path: test-results
- - name: Upload diagnostics
+ run: make ${{ matrix.test }}
+ - name: Upload logs
  if: ${{ failure() && (matrix.test == 'test-e2e' || matrix.test == 'test-e2e-cron') }}
  uses: actions/upload-artifact@v1
  with:
- name: ${{ matrix.test }}-diagnostics
- path: /tmp/log/argo-e2e
+ name: ${{ matrix.test }}-${{ github.run_id }}-argo.log
+ path: /tmp/log/argo-e2e/argo.log
 
  codegen:
  name: Codegen
@@ -114,10 +102,6 @@ jobs:
  run: |
  echo "::add-path::/home/runner/go/bin"
  echo "::add-path::/usr/local/bin"
- - name: Install Kustomize
- run: |
- cd /usr/local/bin && curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | sudo bash
- kustomize version
  - name: Install protoc
  run: |
  set -eux -o pipefail
@@ -130,17 +114,20 @@ jobs:
  sudo find /usr/local/include -type d | xargs sudo chmod a+rx
  rm -f $PROTOC_ZIP
  ls /usr/local/include/google/protobuf/
+ - name: Install `swagger-markdown`
+ run: |
+ sudo chmod a+rw /usr/local/bin /usr/local/lib /usr/local/lib/node_modules
+ make /usr/local/bin/swagger-markdown
  - name: Make codegen
  env:
  GOPATH: /home/runner/go
  run: |
  git fetch --tags
- make codegen
+ make codegen STATIC_FILES=false
  - name: Make lint
  env:
  GOPATH: /home/runner/go
- run: |
- make lint
+ run: make lint STATIC_FILES=false
  - name: Ensure nothing changed
  run: git diff --exit-code
 

.gitignore

@@ -29,5 +29,4 @@ git-ask-pass.sh
 /pkg/apiclient/workflowarchive/workflow-archive.swagger.json
 /pkg/apiclient/workflowtemplate/workflow-template.swagger.json
 /site/
-/node_modules/
 /.brew_home

Makefile

@@ -56,6 +56,7 @@ AUTH_MODE := client
 endif
 K3D := $(shell if [ "`which kubectl`" != '' ] && [ "`kubectl config current-context`" = "k3s-default" ]; then echo true; else echo false; fi)
 LOG_LEVEL := debug
+UPPERIO_DB_DEBUG := 0
 NAMESPACED := true
 
 ALWAYS_OFFLOAD_NODE_STATUS := false
@@ -66,12 +67,6 @@ ifeq ($(PROFILE),postgres)
 ALWAYS_OFFLOAD_NODE_STATUS := true
 endif
 
-ifeq ($(STATIC_FILES),false)
-TEST_OPTS := -coverprofile=coverage.out
-else
-TEST_OPTS :=
-endif
-
 override LDFLAGS += \
  -X github.com/argoproj/argo.version=$(VERSION) \
  -X github.com/argoproj/argo.buildDate=${BUILD_DATE} \
@@ -283,11 +278,9 @@ proto: $(GOPATH)/bin/go-to-protobuf $(GOPATH)/bin/protoc-gen-gogo $(GOPATH)/bin/
  ./hack/generate-proto.sh
  ./hack/update-codegen.sh
 
-dist/install_kustomize.sh:
+/usr/local/bin/kustomize:
  mkdir -p dist
  ./hack/recurl.sh dist/install_kustomize.sh https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh
-
-/usr/local/bin/kustomize: dist/install_kustomize.sh
  chmod +x ./dist/install_kustomize.sh
  ./dist/install_kustomize.sh
  sudo mv kustomize /usr/local/bin/
@@ -321,22 +314,7 @@ endif
 # for local we have a faster target that prints to stdout, does not use json, and can cache because it has no coverage
 .PHONY: test
 test: server/static/files.go
- @mkdir -p test-results
- go test -v $(TEST_OPTS) ./... 2>&1 | tee test-results/test.out
-
-test-results/test-report.json: test-results/test.out
- cat test-results/test.out | go tool test2json > test-results/test-report.json
-
-$(GOPATH)/bin/go-junit-report:
- go get github.com/jstemmer/go-junit-report
-
-# note that we do not have a dependency on test.out, we assume you did correctly create this
-test-results/junit.xml: $(GOPATH)/bin/go-junit-report test-results/test.out
- cat test-results/test.out | go-junit-report > test-results/junit.xml
-
-.PHONY: test-report
-test-report: test-results/junit.xml
- go run ./hack test-report
+ go test ./...
 
 dist/$(PROFILE).yaml: $(MANIFESTS) $(E2E_MANIFESTS) /usr/local/bin/kustomize
  mkdir -p dist
@@ -374,7 +352,7 @@ test-images:
 
 .PHONY: stop
 stop:
- killall argo workflow-controller pf.sh kubectl || true
+ killall argo workflow-controller kubectl || true
 
 $(GOPATH)/bin/goreman:
  go get github.com/mattn/goreman
@@ -391,7 +369,7 @@ endif
  grep '127.0.0.1 *minio' /etc/hosts
  grep '127.0.0.1 *postgres' /etc/hosts
  grep '127.0.0.1 *mysql' /etc/hosts
- env SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start
+ env SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) VERSION=$(VERSION) AUTH_MODE=$(AUTH_MODE) NAMESPACED=$(NAMESPACED) NAMESPACE=$(KUBE_NAMESPACE) $(GOPATH)/bin/goreman -set-ports=false -logtime=false start
 
 .PHONY: wait
 wait:
@@ -411,28 +389,22 @@ mysql-cli:
 
 .PHONY: test-e2e
 test-e2e:
- # Run E2E tests
- @mkdir -p test-results
- go test -timeout 15m -v -count 1 --tags e2e -p 1 --short ./test/e2e 2>&1 | tee test-results/test.out
+ go test -timeout 15m -count 1 --tags e2e -p 1 --short ./test/e2e
 
 .PHONY: test-e2e-cron
 test-e2e-cron:
- # Run E2E tests
- @mkdir -p test-results
- go test -timeout 7m -v -count 1 --tags e2e -parallel 10 -run CronSuite ./test/e2e 2>&1 | tee test-results/test.out
+ go test -count 1 --tags e2e -parallel 10 -run CronSuite ./test/e2e
 
 .PHONY: smoke
 smoke:
- # Run smoke tests
- @mkdir -p test-results
- go test -timeout 1m -v -count 1 --tags e2e -p 1 -run SmokeSuite ./test/e2e 2>&1 | tee test-results/test.out
+ go test -count 1 --tags e2e -p 1 -run SmokeSuite ./test/e2e
 
 # clean
 
 .PHONY: clean
 clean:
- # Delete build files
- rm -Rf vendor dist/* ui/dist
+ go clean
+ rm -Rf test-results node_modules vendor dist/* ui/dist
 
 # swagger
 
@@ -479,13 +451,11 @@ api/openapi-spec/swagger.json: dist/kubeified.swagger.json
  swagger validate api/openapi-spec/swagger.json
  go test ./api/openapi-spec
 
+/usr/local/bin/swagger-markdown:
+ npm install -g swagger-markdown
 
-./node_modules/.bin/swagger-markdown:
- npm init -y
- npm install swagger-markdown
-
-docs/swagger.md: api/openapi-spec/swagger.json ./node_modules/.bin/swagger-markdown
- ./node_modules/.bin/swagger-markdown -i api/openapi-spec/swagger.json -o docs/swagger.md
+docs/swagger.md: api/openapi-spec/swagger.json /usr/local/bin/swagger-markdown
+ swagger-markdown -i api/openapi-spec/swagger.json -o docs/swagger.md
  rm -rf package-lock.json package.json node_modules/
 
 .PHONY: docs

Procfile

@@ -1,2 +1,2 @@
-controller: ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=1 ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image argoproj/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --loglevel ${LOG_LEVEL}
-argo-server: UPPERIO_DB_DEBUG=1 ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE
+controller: ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} OFFLOAD_NODE_STATUS_TTL=30s WORKFLOW_GC_PERIOD=30s UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ARCHIVED_WORKFLOW_GC_PERIOD=30s ./dist/workflow-controller --executor-image argoproj/argoexec:${VERSION} --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --loglevel ${LOG_LEVEL}
+argo-server: UPPERIO_DB_DEBUG=${UPPERIO_DB_DEBUG} ./dist/argo --loglevel ${LOG_LEVEL} server --namespaced=${NAMESPACED} --namespace ${NAMESPACE} --auth-mode ${AUTH_MODE} --secure=$SECURE

docs/running-locally.md

@@ -74,30 +74,6 @@ Or
  make start COMPONENTS=argo-server
 
 To find the command arguments you need to use, you’ll have to look at the `start` target in the `Makefile`.`
-
-### Running Sonar Locally
-
-This can only be done if you have already created a pull request.
-
-Install the scanner:
-
-```
-brew install sonar-scanner
-```
-
-Run the tests:
-
-```
-make test CI=true
-make test-reports/test-report.out
-```
-
-Perform a scan:
-
-```
-# the key is PR number (e.g. "2666"), the branch is the CI branch, e.g. "pull/2666"
-SONAR_TOKEN=... sonar-scanner -Dsonar.pullrequest.key=... -Dsonar.pullrequest.branch=... 
-```
 
 ## Clean
 

docs/static-code-analysis.md

@@ -4,6 +4,5 @@ We use the following static code analysis tools:
 
 * golangci-lint and tslint for compile time linting
 * [snyk.io](https://app.snyk.io/org/argoproj/projects) - for image scanning
-* [sonarcloud.io](https://sonarcloud.io/organizations/argoproj/projects) - for code scans and security alerts
 
 These are at least run daily or on each pull request.

hack/main.go

@@ -18,8 +18,6 @@ func main() {
  secondarySwaggerGen()
  case "parseexamples":
  parseExamples()
- case "test-report":
- testReport()
  default:
  panic(os.Args[1])
  }