Skip to content

Commit

Permalink
ci: Github actions refactor (argoproj#5280)
Browse files Browse the repository at this point in the history
  • Loading branch information
alexec committed Mar 8, 2021
1 parent ba949c3 commit 814944c
Show file tree
Hide file tree
Showing 41 changed files with 1,056 additions and 342 deletions.
211 changes: 44 additions & 167 deletions .github/workflows/ci-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,197 +10,74 @@ on:
- "master"

jobs:

tests:
name: Unit Tests
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Restore go build cache
uses: actions/cache@v1
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }}
- name: Setup Golang
uses: actions/setup-go@v1
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: "1.15.7"
- name: Add bins to PATH
run: |
echo /home/runner/go/bin >> $GITHUB_PATH
echo /usr/local/bin >> $GITHUB_PATH
- name: Run tests
env:
GOPATH: /home/runner/go
run: make test STATIC_FILES=false

e2e-tests:
name: E2E Tests
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
test:
- test-executor
containerRuntimeExecutor:
- docker
- emissary
- k8sapi
- kubelet
- pns
include:
- test: test-cli
containerRuntimeExecutor: docker
- test: test-examples
containerRuntimeExecutor: docker
- test: test-functional
containerRuntimeExecutor: docker
- test: test-e2e-cron
containerRuntimeExecutor: docker
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Restore go build cache
uses: actions/cache@v1
- uses: actions/cache@v2
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }}
- name: Setup Golang
uses: actions/setup-go@v1
path: /home/runner/.cache/go-build
key: GOCACHE-v2-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
go-version: "1.15.7"
- name: Add bins to PATH
run: |
echo /home/runner/go/bin >> $GITHUB_PATH
echo /usr/local/bin >> $GITHUB_PATH
- name: Install and start K3S v1.18.8+k3s1
run: curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.8+k3s1 INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh - &
- name: Pre-pull images
env:
GOPATH: /home/runner/go
run: make test-images &
- name: Create Kubeconfig
run: |
mkdir -p ~/.kube
until stat /etc/rancher/k3s/k3s.yaml ; do sleep 10s ; done
cat /etc/rancher/k3s/k3s.yaml | sed "s/127.0.0.1/$(hostname)/g" > ~/.kube/config
echo "- name: fake_token_user" >> ~/.kube/config
echo " user:" >> ~/.kube/config
echo " token: xxxxxx" >> ~/.kube/config
- name: Start logging
run: |
mkdir -p /tmp/log/argo-e2e
make logs > /tmp/log/argo-e2e/pod.log &
- name: Start Argo
env:
GOPATH: /home/runner/go
run: |
echo '127.0.0.1 dex' | sudo tee -a /etc/hosts
echo '127.0.0.1 minio' | sudo tee -a /etc/hosts
echo '127.0.0.1 postgres' | sudo tee -a /etc/hosts
echo '127.0.0.1 mysql' | sudo tee -a /etc/hosts
git fetch --tags
PROFILE=mysql
ALWAYS_OFFLOAD_NODE_STATUS=true
if [ "${{matrix.test}}" = test-examples ]; then
PROFILE=minimal
ALWAYS_OFFLOAD_NODE_STATUS=false
fi
KUBECONFIG=~/.kube/config make install PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} DEV_IMAGE=true STATIC_FILES=false
KUBECONFIG=~/.kube/config make start PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
- name: Wait for Argo Server to be ready
env:
GOPATH: /home/runner/go
run: make wait
- name: Run tests
env:
GOPATH: /home/runner/go
run: make ${{ matrix.test }}
- name: Upload logs
if: ${{ failure() }}
uses: actions/upload-artifact@v1
with:
name: ${{ matrix.test }}-${{matrix.containerRuntimeExecutor}}-${{ github.run_id }}-pod.log
path: /tmp/log/argo-e2e
path: /home/runner/go/pkg/mod
key: GOMODCACHE-v2-${{ hashFiles('**/go.mod') }}
- run: go mod download -x
- run: make server/static/files.go STATIC_FILES=false
- run: go build -v ./...
- run: make test STATIC_FILES=false

codegen:
name: Codegen
runs-on: ubuntu-latest
timeout-minutes: 15
runs-on: ubuntu-20.04
needs: [ tests ]
timeout-minutes: 10
env:
GOPATH: /home/runner/go
PROTOC_ZIP: protoc-3.11.1-linux-x86_64.zip
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Restore go build cache
uses: actions/cache@v1
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }}
- name: Setup Golang
uses: actions/setup-go@v1
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: "1.15.7"
- name: Add bins to PATH
run: |
echo /home/runner/go/bin >> $GITHUB_PATH
echo /usr/local/bin >> $GITHUB_PATH
- uses: actions/cache@v2
with:
path: /home/runner/.cache/go-build
key: GOCACHE-v2-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
path: /home/runner/go/pkg/mod
key: GOMODCACHE-v2-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
path: /home/runner/go/bin
key: go-bin-v1-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
path: protoc-3.11.1-linux-x86_64.zip
key: protoc-3.11.1-linux-x86_64.zip
- name: Install protoc
run: |
set -eux -o pipefail
PROTOC_ZIP=protoc-3.11.1-linux-x86_64.zip
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.11.1/$PROTOC_ZIP
sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*'
sudo chmod +x /usr/local/bin/protoc
sudo find /usr/local/include -type f | xargs sudo chmod a+r
sudo find /usr/local/include -type d | xargs sudo chmod a+rx
rm -f $PROTOC_ZIP
ls /usr/local/include/google/protobuf/
- name: Make codegen
env:
GOPATH: /home/runner/go
- run: git fetch --tags
- name: Create links
run: |
git fetch --tags
mkdir -p /home/runner/go/src/github.com/argoproj
ln -s "$PWD" /home/runner/go/src/github.com/argoproj/argo-workflows
# we use -B to force make to always make targets
make -B codegen STATIC_FILES=false
- name: Make lint
env:
GOPATH: /home/runner/go
run: make lint STATIC_FILES=false
- name: Make validate-examples
env:
GOPATH: /home/runner/go
run: make validate-examples STATIC_FILES=false
- name: Ensure nothing changed
run: git diff --exit-code

ui:
name: UI
runs-on: ubuntu-latest
timeout-minutes: 8
steps:
- name: Checkout code
uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
node-version: "14"
- name: Restore node dependency cache
uses: actions/cache@v1
with:
path: ui/node_modules
key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
- name: Install, build and Lint
env:
NODE_OPTIONS: --max-old-space-size=4096
run: |
yarn --cwd ui install
yarn --cwd ui build
yarn --cwd ui test
yarn --cwd ui lint
- name: Ensure nothing changed
run: git diff --exit-code
- run: make codegen -B STATIC_FILES=false
- run: make lint STATIC_FILES=false
- run: make validate-examples STATIC_FILES=false
- run: git diff --exit-code
29 changes: 29 additions & 0 deletions .github/workflows/ci-gen.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -eux pipefail

# Github Actions does not provide either:
#
# 1. Any way to re-use YAML.
# 2. Re-run a single job in a workflow.
#
# So this script auto-generates E2E workflows so they can be re-run.

cd $(dirname $0)

gen() {
sed "s/name: E2E/name: $1/" < e2e.yaml.0 |
sed "s/\${{matrix.test}}/$2/g" |
sed "s/\${{matrix.containerRuntimeExecutor}}/$3/g" |
../../hack/auto-gen-msg.sh > $2-$3.yaml
}

gen "Test Docker Executor" test-executor docker
gen "Test Emissary Executor" test-executor emissary
gen "Test K8SAPI Executor" test-executor k8sapi
gen "Test Kubelet Executor" test-executor kubelet
gen "Test PNS Executor" test-executor pns

gen "Test CLI" test-cli docker
gen "Test Cron Workflows" test-e2e-cron docker
gen "Test Examples" test-examples docker
gen "Test Functionality " test-functional docker
10 changes: 5 additions & 5 deletions .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ defaults:
jobs:
build-linux-amd64:
name: Build & push linux/amd64
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
strategy:
matrix:
platform: [linux/amd64]
Expand Down Expand Up @@ -80,7 +80,7 @@ jobs:
build-linux-arm64:
name: Build & push linux/arm64
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
strategy:
matrix:
platform: [linux/arm64]
Expand Down Expand Up @@ -189,7 +189,7 @@ jobs:
push-linux-amd64-images:
name: Push manifest with linux/amd64
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
needs: [build-linux-amd64]
steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -232,7 +232,7 @@ jobs:
push-images:
name: Push manifest with all images
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
needs: [build-linux-arm64, build-windows, push-linux-amd64-images]
steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -279,7 +279,7 @@ jobs:
done
test-images-linux-amd64:
name: Try pulling linux/amd64
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
needs: [push-images]
strategy:
matrix:
Expand Down
75 changes: 75 additions & 0 deletions .github/workflows/e2e.yaml.0
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
name: E2E
on:
push:
branches:
- "master"
- "release-*"
- "!release-2.8"
pull_request:
branches:
- "master"

jobs:

e2e-tests:
name: Test
runs-on: ubuntu-20.04
timeout-minutes: 20
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: "1.15.7"
- uses: actions/cache@v2
with:
path: /home/runner/.cache/go-build
key: GOCACHE-v2-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
path: /home/runner/go/pkg/mod
key: GOMODCACHE-v2-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
path: /home/runner/go/bin
key: go-bin-v1-${{ hashFiles('**/go.mod') }}
- uses: actions/cache@v2
with:
path: dist/kustomize
key: kustomize
- run: make pull-images &
- name: Install and start K3S v1.18.8+k3s1
run: curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.8+k3s1 INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh - &
- name: Create Kubeconfig
run: |
mkdir -p ~/.kube
until stat /etc/rancher/k3s/k3s.yaml ; do sleep 10s ; done
cat /etc/rancher/k3s/k3s.yaml | sed "s/127.0.0.1/$(hostname)/g" > ~/.kube/config
echo "- name: fake_token_user" >> ~/.kube/config
echo " user:" >> ~/.kube/config
echo " token: xxxxxx" >> ~/.kube/config
- run: git fetch --tags
- name: Start Argo
run: |
echo '127.0.0.1 dex' | sudo tee -a /etc/hosts
echo '127.0.0.1 minio' | sudo tee -a /etc/hosts
echo '127.0.0.1 postgres' | sudo tee -a /etc/hosts
echo '127.0.0.1 mysql' | sudo tee -a /etc/hosts
PROFILE=mysql
ALWAYS_OFFLOAD_NODE_STATUS=true
if [ "${{matrix.test}}" = test-executor ] || [ "${{matrix.test}}" = test-examples ]; then
PROFILE=minimal
ALWAYS_OFFLOAD_NODE_STATUS=false
fi
mkdir -p /tmp/log/argo-e2e
KUBECONFIG=~/.kube/config make install PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} STATIC_FILES=false DEV_IMAGE=true
KUBECONFIG=~/.kube/config make start PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=${ALWAYS_OFFLOAD_NODE_STATUS} STATIC_FILES=false DEV_IMAGE=true 2>&1 > /tmp/log/argo-e2e/argo.log &
- run: make wait
- name: Start logging
run: make logs 2>&1 > /tmp/log/argo-e2e/pod.log &
- run: make ${{matrix.test}}
- name: Upload logs
if: ${{ failure() }}
uses: actions/upload-artifact@v1
with:
name: ${{matrix.test}}-${{matrix.containerRuntimeExecutor}}
path: /tmp/log/argo-e2e
Loading

0 comments on commit 814944c

Please sign in to comment.