Use comment trigger to run acceptance tests for PRs. (#929)

* Initial workflow testing.

* Allow running tests for multiple packages concurrently.

* Set PR status checks.

* Cancel pending on new run.

* Run schedualed sweeper after acceptance tests complete.

* Add CONTRIBUTING.md

.github/workflows/acceptance-test-pr.yml

@@ -0,0 +1,71 @@
+on:
+ repository_dispatch:
+ types: [testacc-command]
+
+name: Acceptance Test PR
+
+jobs:
+ acceptance:
+ runs-on: ubuntu-latest
+ if:
+ github.event.client_payload.slash_command.sha != '' &&
+ github.event.client_payload.slash_command.pkg != '' &&
+ github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.sha
+
+ concurrency:
+ group: ${{ github.workflow }}-${{ github.event.client_payload.slash_command.pkg }}-${{ github.event.client_payload.pull_request.number }}
+ cancel-in-progress: true
+
+ steps:
+ - name: Set status pending
+ run: |
+ gh api --method POST -H "Accept: application/vnd.github+json" \
+ /repos/${{ github.repository }}/statuses/${{ github.event.client_payload.pull_request.head.sha }} \
+ -f state='pending' \
+ -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+ -f description='Running acceptance tests...' \
+ -f context='acceptance/${{ github.event.client_payload.slash_command.pkg }}'
+ env:
+ GH_TOKEN: ${{ github.token }}
+
+ - name: Set up Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: 1.19.x
+
+ - name: Checkout PR
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.client_payload.slash_command.sha }}
+
+ - name: Run Acceptance Tests
+ id: run_tests
+ run: make PKG_NAME="${{ github.event.client_payload.slash_command.pkg }}" testacc
+ env:
+ ACCTEST_PARALLELISM: 10
+ DIGITALOCEAN_TOKEN: ${{ secrets.ACCEPTANCE_TESTS_TOKEN }}
+ SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
+ SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
+
+ - name: Results
+ if: always()
+ run: |
+ if [[ ${{ steps.run_tests.outcome }} == 'success' ]]; then
+ echo "test_result=success" >> $GITHUB_ENV
+ echo "check_description=Acceptance tests for ${{ github.event.client_payload.slash_command.pkg }} successful" >> $GITHUB_ENV
+ else
+ echo "test_result=failure" >> $GITHUB_ENV
+ echo "check_description=Acceptance tests for ${{ github.event.client_payload.slash_command.pkg }} failed" >> $GITHUB_ENV
+ fi
+
+ - name: Update status on PR
+ if: always()
+ run: |
+ gh api --method POST -H "Accept: application/vnd.github+json" \
+ /repos/${{ github.repository }}/statuses/${{ github.event.client_payload.pull_request.head.sha }} \
+ -f state='${{ env.test_result }}' \
+ -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+ -f description='${{ env.check_description }}' \
+ -f context='acceptance/${{ github.event.client_payload.slash_command.pkg }}'
+ env:
+ GH_TOKEN: ${{ github.token }}

.github/workflows/acceptance-test-schedule.yml

@@ -16,7 +16,7 @@ jobs:
  - name: Install Go
  uses: actions/setup-go@v2
  with:
- go-version: 1.18.x
+ go-version: 1.19.x
 
  - name: Checkout
  uses: actions/checkout@v2
@@ -30,13 +30,15 @@ jobs:
  SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
 
  sweep:
+ if: ${{ always() }}
+ needs: acceptance
  runs-on: ubuntu-latest
 
  steps:
  - name: Install Go
  uses: actions/setup-go@v2
  with:
- go-version: 1.18.x
+ go-version: 1.19.x
 
  - name: Checkout
  uses: actions/checkout@v2

.github/workflows/acceptance-test-trigger.yml

@@ -0,0 +1,19 @@
+name: Acceptance Test Trigger
+
+on:
+ issue_comment:
+ types: [created]
+
+jobs:
+ testacc-trigger:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Slash Command Dispatch
+ uses: peter-evans/slash-command-dispatch@v1
+ with:
+ token: ${{ secrets.TRIGGER_TOKEN }}
+ issue-type: pull-request
+ commands: testacc
+ named-args: true
+ permission: write

.github/workflows/test.yml

@@ -38,57 +38,3 @@ jobs:
 
  - name: make test
  run: make test
-
- acceptance:
- # Runs the acceptance test suite using the `Acceptance Tests` environment.
- # The enviroment should be configured to require reviewers approve the
- # step. This allows the tests to be added as a check to PRs.
- #
- # This job has the `continue-on-error` set to true which prevents blocking
- # PRs if the tests fail.
- needs: unit
- runs-on: ubuntu-latest
- environment:
- name: Acceptance Tests
-
- concurrency: acceptance_tests
-
- continue-on-error: true
-
- steps:
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19.x
-
- - name: Checkout
- uses: actions/checkout@v2
-
- - name: make testacc
- run: make testacc
- env:
- TESTARGS: -parallel 20
- DIGITALOCEAN_TOKEN: ${{ secrets.ACCEPTANCE_TESTS_TOKEN }}
- SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
- SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}
-
- sweep:
- runs-on: ubuntu-latest
- needs: acceptance
- concurrency: acceptance_tests
-
- steps:
- - name: Install Go
- uses: actions/setup-go@v2
- with:
- go-version: 1.19.x
-
- - name: Checkout
- uses: actions/checkout@v2
-
- - name: make sweep
- run: make sweep
- env:
- DIGITALOCEAN_TOKEN: ${{ secrets.ACCEPTANCE_TESTS_TOKEN }}
- SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY_ID }}
- SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_ACCESS_KEY }}

CONTRIBUTING.md

@@ -0,0 +1,126 @@
+Developing the Provider
+---------------------------
+
+If you wish to work on the provider, you'll first need [Go](http:https://www.golang.org) installed on your machine (version 1.11+ is *required*). You'll also need to correctly setup a [GOPATH](http:https://golang.org/doc/code.html#GOPATH), as well as adding `$GOPATH/bin` to your `$PATH`.
+
+To compile the provider, run `make build`. This will build the provider and put the provider binary in the `$GOPATH/bin` directory.
+
+```sh
+$ make build
+...
+$ $GOPATH/bin/terraform-provider-digitalocean
+...
+```
+
+In order to test the provider, you can simply run `make test`.
+
+```sh
+$ make test
+```
+
+In order to run the full suite of acceptance tests, run `make testacc`.
+
+*Note:* Acceptance tests create real resources, and often cost money to run.
+
+```sh
+$ make testacc
+```
+
+In order to run a specific acceptance test, use the `TESTARGS` environment variable. For example, the following command will run `TestAccDigitalOceanDomain_Basic` acceptance test only:
+
+```sh
+$ make testacc TESTARGS='-run=TestAccDigitalOceanDomain_Basic'
+```
+
+All acceptance tests for a specific package can be run by setting the `PKG_NAME` environment variable. For example:
+
+```sh
+$ make testacc PKG_NAME=digitalocean/account
+```
+
+In order to check changes you made locally to the provider, you can use the binary you just compiled by adding the following
+to your `~/.terraformrc` file. This is valid for Terraform 0.14+. Please see
+[Terraform's documentation](https://www.terraform.io/docs/cli/config/config-file.html#development-overrides-for-provider-developers)
+for more details.
+
+```
+provider_installation {
+
+ # Use /home/developer/go/bin as an overridden package directory
+ # for the digitalocean/digitalocean provider. This disables the version and checksum
+ # verifications for this provider and forces Terraform to look for the
+ # digitalocean provider plugin in the given directory.
+ dev_overrides {
+ "digitalocean/digitalocean" = "/home/developer/go/bin"
+ }
+
+ # For all other providers, install them directly from their origin provider
+ # registries as normal. If you omit this, Terraform will _only_ use
+ # the dev_overrides block, and so no other providers will be available.
+ direct {}
+}
+```
+
+For information about writing acceptance tests, see the main Terraform [contributing guide](https://github.com/hashicorp/terraform/blob/master/.github/CONTRIBUTING.md#writing-acceptance-tests).
+
+Releasing the Provider
+----------------------
+
+To release the provider:
+
+1. Use
+ [github-changelog-generator](https://github.com/digitalocean/github-changelog-generator)
+ to list the changes since the last release and decide what kind of release
+ you are doing (bugfix, feature or breaking).
+1. Create a new commit that only contains updates to
+ [CHANGELOG.md](CHANGELOG.md) listing the respective changes for the new
+ version. Godo follows [semver](https://www.semver.org/) versioning
+ semantics.
+1. Once the CHANGELOG.md commit is merged, create a new tag on that commit with
+ the new version that will be released (be sure to pull the latest from
+ git).
+
+ ```bash
+ git tag -m "release $new_version" -a "$new_version"
+ ```
+
+1. Push the tag:
+
+ ```bash
+ git push "$origin" tag "$new_version"
+ ```
+
+This repository contains a GitHub Action configured to automatically build and
+publish assets for release when a tag is pushed that matches the pattern `v*`
+(ie. `v0.1.0`).
+
+A [Goreleaser](https://goreleaser.com/) configuration is provided that produces
+build artifacts matching the [layout required](https://www.terraform.io/docs/registry/providers/publishing.html#manually-preparing-a-release)
+to publish the provider in the Terraform Registry.
+
+Releases will appear as drafts. Once marked as published on the GitHub Releases page,
+they will become available via the Terraform Registry.
+
+Reviewing Pull Requests
+-----------------------
+
+Acceptance tests use the production API and create resources that incur costs.
+Running the full suite of acceptance tests can also take quite a long time to run.
+In order to prevent misuse, the acceptance tests for a pull request must be manually
+triggered by a reviewer with write access to the repository.
+
+To trigger a run of the acceptance tests for a PR, you may use the `/testacc` in a
+comment. The `pkg` and `sha` arguments are required. This allows us to limit the
+packages being tested for quick feedback and protect against timing attacks.
+For example, to run the acceptance tests for the `droplet` package, the command
+may look like:
+
+ /testacc pkg=digitalocean/droplet sha=d358bd2418b4e30d7bdf2b98b4c151e357814c63
+
+To run the entire suite, use `pkg=digitalocean`.
+
+If multiple packages are to be tested, each command must be posted as a separate
+comment. Only the first line of the comment is evaluated.
+
+We leverage the [`peter-evans/slash-command-dispatch`](https://github.com/peter-evans/slash-command-dispatch)
+GitHub Action for the command processing.

GNUmakefile

@@ -1,8 +1,8 @@
 TEST?=$$(go list ./... |grep -v 'vendor')
 GOFMT_FILES?=$$(find . -name '*.go' |grep -v vendor)
-PKG_NAME:=digitalocean
-ACCTEST_TIMEOUT:=120m
-ACCTEST_PARALLELISM:=2
+PKG_NAME?=digitalocean
+ACCTEST_TIMEOUT?=120m
+ACCTEST_PARALLELISM?=2
 
 default: build