Add tvm-bot lambda (#52)

This moves the tvm-bot pr commenting logic from apache/tvm to an AWS Lambda function. This keeps it out of the queue and also gives us better guarantees about runtime (PR comments previously would take up to a minute to show up, with this they are pretty much instant), and this doesn't show up in the CI jobs box on PRs.

There is lots of copy paste from `tvm/ci/scripts` into `tvm_bot/tvm_bot` with some minor changes to work with lambda. The main infra changes are in the folder atop that in `ci/terraform`, which adds a lambda function with an API gateway that exposes a URL that GitHub can send webhooks to.

This PR is just a start so it skips the more complicated parts that generate PR-specific bits, like the docs URL and the skipped tests list. The commenting is also still gated to members of the OSS team.

.github/workflows/jenkins_deploy.yml

@@ -1,12 +1,12 @@
 name: deploy_jenkins
-on: 
+on:
  workflow_dispatch:
 
 concurrency:
  group: deploy_jenkins
  cancel-in-progress: true
 
-jobs: 
+jobs:
  deploy_docker:
  runs-on: ubuntu-latest
  if: github.ref == 'refs/heads/main'
@@ -37,6 +37,6 @@ jobs:
  -
  name: Set permissions on key file
  run: chmod 600 global_node_access_private_key
- - 
+ -
  name: Deploy docker images to head node
  run: ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i ansible_inventory.txt -u ubuntu -b --private-key=global_node_access_private_key ansible/playbooks/deploy_docker.yaml

.github/workflows/jenkins_prepare.yml

@@ -82,5 +82,3 @@ jobs:
  -
  name: Run Ansible playbook
  run: ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i ansible_inventory.txt -u ubuntu -b --private-key=global_node_access_private_key ansible/playbooks/jenkins.yaml
-
-

.github/workflows/packer_build.yml

@@ -27,7 +27,7 @@ jobs:
  -
  name: Pull repository
  uses: actions/checkout@v2
- - 
+ -
  name: Build AWS base AMI
  uses: hashicorp/packer-github-actions@master
  env:
@@ -36,7 +36,7 @@ jobs:
  with:
  command: build
  target: packer/base-images/aws
- - 
+ -
  name: Build stock Jenkins agent
  uses: hashicorp/packer-github-actions@master
  env:
@@ -46,7 +46,7 @@ jobs:
  with:
  command: build
  target: packer/jenkins-agents/stock
- - 
+ -
  name: Build GPU Jenkins agent
  uses: hashicorp/packer-github-actions@master
  env:

.github/workflows/terraform_apply.yml

@@ -18,6 +18,10 @@ jobs:
  AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TF_VAR_tvm_bot_webhook_secret: ${{ secrets.TVM_BOT_WEBHOOK_SECRET }}
+ TF_VAR_tvm_bot_github_token: ${{ secrets.TVM_BOT_GITHUB_TOKEN }}
+ TF_VAR_tvm_bot_repo: tvm
+ TF_VAR_tvm_bot_owner: apache
  defaults:
  run:
  working-directory: ./terraform

.github/workflows/terraform_plan.yml

@@ -52,13 +52,18 @@ jobs:
  AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ TF_VAR_tvm_bot_webhook_secret: ${{ secrets.TVM_BOT_WEBHOOK_SECRET }}
+ TF_VAR_tvm_bot_github_token: ${{ secrets.TVM_BOT_GITHUB_TOKEN }}
+ TF_VAR_tvm_bot_repo: tvm
+ TF_VAR_tvm_bot_owner: apache
  runs-on: ubuntu-latest
  defaults:
  run:
  working-directory: ./terraform
  needs: validate
  if: needs.validate.outputs.valid_workflow == 'True'
- #These steps run if either the PR is within the same repo or if the PR is on a fork and the committer has deployer access
+ # These steps run if either the PR is within the same repo or if the PR is
+ # on a fork and the committer has deployer access
  steps:
  - uses: actions/checkout@v2
  with:

.github/workflows/test.yml

@@ -0,0 +1,33 @@
+name: test tvm_bot
+on:
+ push:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - terraform/tvm_bot/**
+ pull_request_target:
+ branches:
+ - main
+ paths:
+ - terraform/tvm_bot/**
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: ./terraform/tvm_bot
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.9'
+ - name: Install dependencies
+ run: |
+ set -eux
+ pip install -r requirements.txt
+ - name: Run unit tests
+ run: |
+ set -eux
+ PYTHONPATH=$(pwd) pytest --tb=native

terraform/.gitignore

@@ -19,3 +19,4 @@ coverage.xml
 .pytest_cache/
 .testmondata
 
+*.env

terraform/README.md

@@ -1,3 +1,24 @@
 # terraform
 
 This folder handles the Terraform configuration for TVM Jenkins Infrastructure.
+
+## Local Usage
+
+```bash
+# if anything is broken, remove all terraform local files
+git clean -xfd .
+
+# set credentials
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+
+# get terraform state
+terraform init
+
+# the workspace must be selected or else 'plan' will not read the correct state
+terraform workspace new tvm-ci-prod
+terraform workspace select tvm-ci-prod
+
+# run the actual plan against AWS
+terraform plan -var-file=vars/tvm-ci-prod.auto.tfvars
+```