all: migrate the rest of the CircleCI jobs to GitHub Actions (stellar…

…#4250)

.github/actions/setup-go/action.yml

@@ -18,7 +18,7 @@ runs:
  shell: bash
  run: |
  # Set a base, fixed modification time of all directories.
- # git-restore-mtime doesn't set the mtime of of all directories.
+ # git-restore-mtime doesn't set the mtime of all directories.
  # (see https://github.com/MestreLion/git-tools/issues/47 for details)
  touch -m -t '201509301646' $(find . -type d -not -path '.git/*')
  # Restore original modification time from git. git clone sets the
@@ -42,7 +42,7 @@ runs:
  # rebuilds and cause test runs to skip tests that have no reason to rerun.
  # Don't run this for protected branches like master/main.
  - uses: actions/cache@v2
- if: ${{ github.ref_protected || github.ref != 'master' || github.ref != 'main' }}
+ if: github.ref_protected || github.ref != 'refs/heads/master'
  with:
  path: ~/.cache/go-build
  key: ${{ env.PREFIX }}-go-build-${{ github.ref }}-${{ hashFiles('**', '!.git') }}

.github/workflows/go.yml

@@ -32,6 +32,9 @@ jobs:
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v2
+ with:
+ # For pull requests, build and test the PR head not a merge of the PR with the destination.
+ ref: ${{ github.head_ref || github.ref }}
  - uses: ./.github/actions/setup-go
  with:
  go-version: ${{ matrix.go }}
@@ -78,6 +81,8 @@ jobs:
  steps:
  - uses: actions/checkout@v2
  with:
+ # For pull requests, build and test the PR head not a merge of the PR with the destination.
+ ref: ${{ github.head_ref || github.ref }}
  # We need to full history for git-restore-mtime to know what modification dates to use.
  # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
  fetch-depth: '0'

.github/workflows/horizon-master.yml

@@ -0,0 +1,54 @@
+name: Horizon master
+
+on:
+ push:
+ branches: [master]
+
+jobs:
+
+ push-horizon-image-sha:
+ name: Push stellar/horizon:sha to DockerHub
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Get image tag (short sha)
+ shell: bash
+ id: get_tag
+ run: echo ::set-output name=TAG::$(git rev-parse --short ${{ github.sha }} )
+
+ - name: Login to DockerHub
+ uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Build and push to DockerHub
+ uses: docker/build-push-action@7f9d37fa544684fb73bfe4835ed7214c255ce02b
+ with:
+ # TODO: Commented out until we disable the CircleCI jobs
+ # push: true
+ tags: stellar/horizon:${{ steps.get_tag.outputs.TAG }}
+ file: services/horizon/docker/Dockerfile.dev
+
+ push-state-diff-image:
+ name: Push stellar/ledger-state-diff:{sha,latest} to DockerHub
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Login to DockerHub
+ uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Build and push to DockerHub
+ uses: docker/build-push-action@7f9d37fa544684fb73bfe4835ed7214c255ce02b
+ with:
+ # TODO: Commented out until we disable the CircleCI jobs
+ # push: true
+ tags: stellar/ledger-state-diff:${{ github.sha }},stellar/ledger-state-diff:latest
+ file: exp/tools/dump-ledger-state/Dockerfile
+ build-args: GITCOMMIT=${{ github.sha }}
+ no-cache: true

.github/workflows/horizon-release.yml

@@ -0,0 +1,43 @@
+name: Horizon release
+
+on:
+ push:
+ tags: ['horizon-v*']
+
+jobs:
+
+ publish-artifacts:
+ runs-on: ubuntu-latest
+ name: Upload artifacts to GitHub release
+ steps:
+ - name: Run deprecation tests
+ shell: bash
+ run: |
+ export VERSION=${GITHUB_REF_NAME#horizon-v}
+ echo "Searching for \"Action needed in release ${VERSION}\" tags..."
+ # Negate the result so process exits with 1 if anything found
+ ! egrep -irn -A 1 --include=*.go "Action.+needed.+in.+release:.+$VERSION" ./
+
+ - uses: actions/checkout@v2
+
+ - uses: ./.github/actions/setup-go
+ with:
+ go-version: 1.17
+
+ - name: Check dependencies
+ run: ./gomod.sh
+
+ - name: Build binaries
+ run: go run ./support/scripts/build_release_artifacts/main.go
+
+ # TODO: Disabled until we remove the CircleCI jobs
+ - if: false
+ name: Upload binaries to release
+ uses: svenstaro/upload-release-action@483c1e56f95e88835747b1c7c60581215016cbf2
+ with:
+ repo_token: ${{ secrets.GITHUB_TOKEN }}
+ tag: ${{ github.ref_name }}
+ file: ./dist/*
+ file_glob: true
+ overwrite: true
+

.github/workflows/horizon.yml

@@ -8,6 +8,7 @@ on:
 jobs:
 
  integration:
+ name: Integration tests
  strategy:
  matrix:
  os: [ubuntu-latest]
@@ -40,14 +41,19 @@ jobs:
  steps:
  - uses: actions/checkout@v2
  with:
+ # For pull requests, build and test the PR head not a merge of the PR with the destination.
+ ref: ${{ github.head_ref || github.ref }}
  # We need to full history for git-restore-mtime to know what modification dates to use.
  # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
  fetch-depth: '0'
+
  - uses: ./.github/actions/setup-go
  with:
  go-version: ${{ matrix.go }}
+
  - name: Pull latest Stellar Core image
  run: docker pull stellar/stellar-core
+
  - if: ${{ startsWith(matrix.ingestion-backend, 'captive-core') }}
  name: Install and enable Captive Core
  run: |
@@ -57,8 +63,37 @@ jobs:
  echo "Using stellar core version $(stellar-core version)"
  echo 'HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true' >> $GITHUB_ENV
  echo 'CAPTIVE_CORE_BIN=/usr/bin/stellar-core' >> $GITHUB_ENV
+
  - if: ${{ matrix.ingestion-backend == 'captive-core-remote-storage' }}
  name: Setup Captive Core Remote Storage
  run: echo 'HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE_USE_DB=true' >> $GITHUB_ENV
+
  - run: go test -race -timeout 25m -v ./services/horizon/internal/integration/...
 
+ verify-range:
+ name: Test (and push) verify-range image
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ # For pull requests, build and test the PR head not a merge of the PR with the destination.
+ ref: ${{ github.head_ref || github.ref }}
+
+ - name: Build and test the Verify Range Docker image
+ run: |
+ docker build -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
+ # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
+ docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
+
+ # Push image
+ - if: github.ref == 'refs/heads/master'
+ name: Login to DockerHub
+ uses: docker/login-action@bb984efc561711aaa26e433c32c3521176eae55b
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - if: github.ref == 'refs/heads/master'
+ name: Push to DockerHub
+ # TODO(fons): Commented out until we disable the CircleCI jobs
+ run: \# docker push stellar/horizon-verify-range:latest
+

services/horizon/docker/verify-range/dependencies

@@ -11,7 +11,7 @@ echo "deb https://apt.stellar.org $(lsb_release -cs) unstable" | sudo tee -a /et
 apt-get update
 apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 
-wget https://dl.google.com/go/go1.17.linux-amd64.tar.gz
+wget -q https://dl.google.com/go/go1.17.linux-amd64.tar.gz
 tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
 
 # configure postgres

support/scripts/build_release_artifacts/main.go

@@ -43,8 +43,11 @@ func main() {
  if os.Getenv("TRAVIS_EVENT_TYPE") == "cron" {
  buildNightlies()
  os.Exit(0)
- } else if os.Getenv("CIRCLE_TAG") != "" {
- buildByTag()
+ } else if circleTag := os.Getenv("CIRCLE_TAG"); circleTag != "" {
+ buildByTag(circleTag)
+ os.Exit(0)
+ } else if ghTag := getGitHubTagName(); ghTag != "" {
+ buildByTag(ghTag)
  os.Exit(0)
  } else {
  buildSnapshots()
@@ -54,6 +57,20 @@ func main() {
  log.Info("nothing to do")
 }
 
+func getGitHubTagName() string {
+ const githubTagRefPrefix = "refs/tags/"
+ ref := os.Getenv("GITHUB_REF")
+ if ref == "" {
+ // we are not in a github action
+ return ""
+ }
+ if !strings.HasPrefix(ref, githubTagRefPrefix) {
+ // we are not in a tag-triggered event
+ return ""
+ }
+ return strings.TrimPrefix(ref, githubTagRefPrefix)
+}
+
 // package searches the `tools` and `services` packages of this repo to find
 // the source directory. This is used within the script to find the README and
 // other files that should be packaged with the binary.
@@ -145,16 +162,16 @@ func buildNightlies() {
  }
 }
 
-func buildByTag() {
- bin, version := extractFromTag(os.Getenv("CIRCLE_TAG"))
- pkg := packageName(bin)
- repo := repoName()
-
+func buildByTag(tag string) {
+ bin, version := extractFromTag(tag)
  if bin == "" {
- log.Info("could not extract info from CIRCLE_TAG: skipping artifact packaging")
+ log.Infof("non-conformant tag name %q: skipping artifact packaging", tag)
  os.Exit(0)
  }
 
+ pkg := packageName(bin)
+ repo := repoName()
+
  // Don't build anything if no package can be found
  if pkg == "" {
  log.Infof("could not find `%s` in expected binary locations: skipping artifact packaging", bin)
@@ -207,12 +224,11 @@ func buildSnapshots() {
 
 // extractFromTag extracts the name of the binary that should be packaged in the
 // course of execution this script as well as the version it should be packaged
-// as, based on the name of the tag in the CIRCLE_TAG environment variable.
+// as, based on the name of the tag.
 // Tags must be of the form `NAME-vSEMVER`, such as `horizon-v1.0.0` to be
 // matched by this function.
 //
-// In the event that the CIRCLE_TAG is missing or the match fails, an empty
-// string will be returned.
+// In the event the match fails, an empty string will be returned.
 func extractFromTag(tag string) (string, string) {
  match := extractBinName.FindStringSubmatch(tag)
  if match == nil {