feat: Adds support for MySQL. Fixes argoproj#1945 (argoproj#2013)

highb · Jan 22, 2020 · 4a1307c · 4a1307c
1 parent d843e60
commit 4a1307c
Show file tree

Hide file tree

Showing 29 changed files with 986 additions and 219 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -47,44 +47,7 @@ commands:
  - go-v5-master-{{ .Environment.CIRCLE_JOB }}
  - go-v4-{{ .Branch }}-{{ .Environment.CIRCLE_JOB }}
  - go-v4-master-{{ .Environment.CIRCLE_JOB }}
- go_junit_report:
- steps:
- - run:
- name: Install Go test report to Junit test report
- command: go get github.com/jstemmer/go-junit-report
-jobs:
- test:
- working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo
- machine:
- image: ubuntu-1604:201903-01
- steps:
- - restore_go_cache
- - install_golang
- - go_junit_report
- - checkout
- - dep_ensure
- - run:
- name: Run tests
- command: |
- mkdir -p test-results
- trap 'go-junit-report < test-results/test.out > test-results/junit.xml' EXIT
- make test 2>&1 | tee test-results/test.out
- - run:
- name: Verify code generation
- command: make verify-codegen
- - save_go_cache
- - run:
- name: Uploading code coverage
- command: bash <(curl -s https://codecov.io/bash) -f coverage.out
- - store_test_results:
- path: test-results
- - store_artifacts:
- path: test-results
- destination: .
  e2e:
- working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo
- machine:
- image: ubuntu-1604:201903-01
  steps:
  - run:
  name: Install Kustomize
@@ -121,7 +84,7 @@ jobs:
  background: true
  - run:
  name: Start Argo
- command: mkdir dist && KUBECONFIG=~/.kube/config make start E2E_EXECUTOR=docker
+ command: mkdir dist && KUBECONFIG=~/.kube/config make start DB=$DB E2E_EXECUTOR=docker
  - run:
  name: Establish port forward
  command: make pf
@@ -145,6 +108,56 @@ jobs:
  - store_artifacts:
  path: test-results
  destination: .
+ go_junit_report:
+ steps:
+ - run:
+ name: Install Go test report to Junit test report
+ command: go get github.com/jstemmer/go-junit-report
+jobs:
+ test:
+ working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo
+ machine:
+ image: ubuntu-1604:201903-01
+ steps:
+ - restore_go_cache
+ - install_golang
+ - go_junit_report
+ - checkout
+ - dep_ensure
+ - run:
+ name: Run tests
+ command: |
+ mkdir -p test-results
+ trap 'go-junit-report < test-results/test.out > test-results/junit.xml' EXIT
+ make test 2>&1 | tee test-results/test.out
+ - run:
+ name: Verify code generation
+ command: make verify-codegen
+ - save_go_cache
+ - run:
+ name: Uploading code coverage
+ command: bash <(curl -s https://codecov.io/bash) -f coverage.out
+ - store_test_results:
+ path: test-results
+ - store_artifacts:
+ path: test-results
+ destination: .
+ e2e-postgres:
+ working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo
+ machine:
+ image: ubuntu-1604:201903-01
+ environment:
+ DB: postgres
+ steps:
+ - e2e
+ e2e-mysql:
+ working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo
+ machine:
+ image: ubuntu-1604:201903-01
+ environment:
+ DB: mysql
+ steps:
+ - e2e
  docker-build:
  working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo
  machine:
@@ -184,5 +197,6 @@ workflows:
  jobs:
  - test
  - ui
- - e2e
+ - e2e-postgres
+ - e2e-mysql
  - docker-build
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Makefile b/Makefile
@@ -1,4 +1,3 @@
-PACKAGE := github.com/argoproj/argo
 
 BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
 GIT_COMMIT = $(shell git rev-parse HEAD)
@@ -28,6 +27,7 @@ endif
 # perform static compilation
 STATIC_BUILD ?= true
 CI ?= false
+DB ?= postgres
 K3D := $(shell if [ "`kubectl config current-context`" = "k3s-default" ]; then echo true; else echo false; fi)
 
 override LDFLAGS += \
@@ -55,7 +55,7 @@ E2E_EXECUTOR ?= pns
 .PHONY: build
 build: clis executor-image controller-image manifests/install.yaml manifests/namespace-install.yaml manifests/quick-start-postgres.yaml manifests/quick-start-mysql.yaml
 
-vendor: Gopkg.toml
+vendor: Gopkg.toml Gopkg.lock
  # Get Go dependencies
  rm -Rf .vendor-new
  dep ensure -v
@@ -205,11 +205,11 @@ manifests/namespace-install.yaml: $(MANIFESTS)
 
 manifests/quick-start-mysql.yaml: $(MANIFESTS)
  # Create MySQL quick-start manifests
- kustomize build manifests/quick-start/mysql > manifests/quick-start-mysql.yaml
+ kustomize build manifests/quick-start/mysql | ./hack/auto-gen-msg.sh > manifests/quick-start-mysql.yaml
 
 manifests/quick-start-postgres.yaml: $(MANIFESTS)
  # Create Postgres quick-start manifests
- kustomize build manifests/quick-start/postgres > manifests/quick-start-postgres.yaml
+ kustomize build manifests/quick-start/postgres | ./hack/auto-gen-msg.sh > manifests/quick-start-postgres.yaml
 
 # lint/test/etc
 
@@ -233,28 +233,45 @@ endif
 
 test/e2e/manifests/postgres.yaml: $(MANIFESTS) $(E2E_MANIFESTS)
  # Create Postgres e2e manifests
- kustomize build test/e2e/manifests/postgres > test/e2e/manifests/postgres.yaml
+ kustomize build test/e2e/manifests/postgres | ./hack/auto-gen-msg.sh > test/e2e/manifests/postgres.yaml
 
 dist/postgres.yaml: test/e2e/manifests/postgres.yaml
  # Create Postgres e2e manifests
  cat test/e2e/manifests/postgres.yaml | sed 's/:latest/:$(IMAGE_TAG)/' | sed 's/pns/$(E2E_EXECUTOR)/' > dist/postgres.yaml
 
-.PHONY: install-postgres
-install-postgres: dist/postgres.yaml
+test/e2e/manifests/mysql/overlays/argo-server-deployment.yaml: test/e2e/manifests/postgres/overlays/argo-server-deployment.yaml
+test/e2e/manifests/mysql/overlays/argo-server-deployment.yaml:
+ cat test/e2e/manifests/postgres/overlays/argo-server-deployment.yaml | ./hack/auto-gen-msg.sh > test/e2e/manifests/mysql/overlays/argo-server-deployment.yaml
+
+test/e2e/manifests/mysql/overlays/workflow-controller-deployment.yaml: test/e2e/manifests/postgres/overlays/workflow-controller-deployment.yaml
+test/e2e/manifests/mysql/overlays/workflow-controller-deployment.yaml:
+ cat test/e2e/manifests/postgres/overlays/workflow-controller-deployment.yaml | ./hack/auto-gen-msg.sh > test/e2e/manifests/mysql/overlays/workflow-controller-deployment.yaml
+
+test/e2e/manifests/mysql.yaml: $(MANIFESTS) $(E2E_MANIFESTS) test/e2e/manifests/mysql/overlays/argo-server-deployment.yaml test/e2e/manifests/mysql/overlays/workflow-controller-deployment.yaml
+ # Create MySQL e2e manifests
+ kustomize build test/e2e/manifests/mysql | ./hack/auto-gen-msg.sh > test/e2e/manifests/mysql.yaml
+
+dist/mysql.yaml: test/e2e/manifests/mysql.yaml
+ # Create MySQL e2e manifests
+ cat test/e2e/manifests/mysql.yaml | sed 's/:latest/:$(IMAGE_TAG)/' | sed 's/pns/$(E2E_EXECUTOR)/' > dist/mysql.yaml
+
+.PHONY: install
+install: dist/postgres.yaml dist/mysql.yaml
  # Install Postgres quick-start
  kubectl get ns argo || kubectl create ns argo
+ifeq ($(DB),postgres)
  kubectl -n argo apply -f dist/postgres.yaml
-
-.PHONY: install
-install: install-postgres
+else
+ kubectl -n argo apply -f dist/mysql.yaml
+endif
 
 .PHONY: start
 start: controller-image cli-image executor-image install
  # Start development environment
 ifeq ($(CI),false)
  make down
- make up
 endif
+ make up
  # Make the CLI
  make cli
  # Switch to "argo" ns.
@@ -290,7 +307,7 @@ pf-bg:
 .PHONY: logs
 logs:
  # Tail logs
- kubectl -n argo logs -f -l app --max-log-requests 10
+ kubectl -n argo logs -f -l app --max-log-requests 10 --tail 100
 
 .PHONY: postgres-cli
 postgres-cli:

diff --git a/hack/auto-gen-msg.sh b/hack/auto-gen-msg.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+echo "# This is an auto-generated file. DO NOT EDIT"
+cat
diff --git a/hack/update-manifests.sh b/hack/update-manifests.sh
@@ -2,8 +2,6 @@
 set -eu -o pipefail
 
 SRCROOT="$( CDPATH='' cd -- "$(dirname "$0")/.." && pwd -P )"
-AUTOGENMSG="# This is an auto-generated file. DO NOT EDIT"
-
 IMAGE_NAMESPACE="${IMAGE_NAMESPACE:-argoproj}"
 VERSION="${VERSION:-latest}"
 
@@ -15,12 +13,10 @@ cd ${SRCROOT}/manifests/base && kustomize edit set image \
  argoproj/workflow-controller=${IMAGE_NAMESPACE}/workflow-controller:${VERSION} \
  argoproj/argocli=${IMAGE_NAMESPACE}/argocli:${VERSION}
 
-echo "${AUTOGENMSG}" > "${SRCROOT}/manifests/install.yaml"
-kustomize build "${SRCROOT}/manifests/cluster-install" >> "${SRCROOT}/manifests/install.yaml"
+kustomize build "${SRCROOT}/manifests/cluster-install" | ${SRCROOT}/hack/auto-gen-msg.sh > "${SRCROOT}/manifests/install.yaml"
 sed -i.bak "s@- .*/argoexec:.*@- ${IMAGE_NAMESPACE}/argoexec:${VERSION}@" "${SRCROOT}/manifests/install.yaml"
 rm -f "${SRCROOT}/manifests/install.yaml.bak"
 
-echo "${AUTOGENMSG}" > "${SRCROOT}/manifests/namespace-install.yaml"
-kustomize build "${SRCROOT}/manifests/namespace-install" >> "${SRCROOT}/manifests/namespace-install.yaml"
+kustomize build "${SRCROOT}/manifests/namespace-install" | ${SRCROOT}/hack/auto-gen-msg.sh > "${SRCROOT}/manifests/namespace-install.yaml"
 sed -i.bak "s@- .*/argoexec:.*@- ${IMAGE_NAMESPACE}/argoexec:${VERSION}@" "${SRCROOT}/manifests/namespace-install.yaml"
 rm -f "${SRCROOT}/manifests/namespace-install.yaml.bak"
diff --git a/manifests/quick-start-mysql.yaml b/manifests/quick-start-mysql.yaml
@@ -1,3 +1,4 @@
+# This is an auto-generated file. DO NOT EDIT
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -141,6 +142,12 @@ rules:
  - update
  - patch
  - delete
+- apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

diff --git a/manifests/quick-start-postgres.yaml b/manifests/quick-start-postgres.yaml
@@ -1,3 +1,4 @@
+# This is an auto-generated file. DO NOT EDIT
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -141,6 +142,12 @@ rules:
  - update
  - patch
  - delete
+- apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

diff --git a/persist/sqldb/ansi_sql_change.go b/persist/sqldb/ansi_sql_change.go
@@ -5,7 +5,7 @@ import "upper.io/db.v3/lib/sqlbuilder"
 // represent a straight forward change that is compatible with all database providers
 type ansiSQLChange string
 
-func (s ansiSQLChange) Apply(session sqlbuilder.Database) error {
+func (s ansiSQLChange) apply(session sqlbuilder.Database) error {
  _, err := session.Exec(string(s))
  return err
 }
diff --git a/persist/sqldb/backfill_cluster_name.go b/persist/sqldb/backfill_cluster_name.go
@@ -17,7 +17,7 @@ func (s backfillClusterName) String() string {
  return fmt.Sprintf("backfillClusterName{%s,%s}", s.clusterName, s.tableName)
 }
 
-func (s backfillClusterName) Apply(session sqlbuilder.Database) error {
+func (s backfillClusterName) apply(session sqlbuilder.Database) error {
  log.WithField("clustername", s.clusterName).Info("Back-filling cluster name")
  rs, err := session.
  Select("uid").

diff --git a/persist/sqldb/backfill_nodes.go b/persist/sqldb/backfill_nodes.go
@@ -19,7 +19,7 @@ func (s backfillNodes) String() string {
  return fmt.Sprintf("backfillNodes{%s}", s.tableName)
 }
 
-func (s backfillNodes) Apply(session sqlbuilder.Database) error {
+func (s backfillNodes) apply(session sqlbuilder.Database) error {
  log.Info("Backfill node status")
  rs, err := session.SelectFrom(s.tableName).
  Columns("workflow").