all: cache go module downloads and other build and test artifacts (#3727

)

.github/actions/setup-go/action.yml

@@ -0,0 +1,51 @@
+name: 'Setup the Go environment'
+description: 'Installs go and restores/saves the build/module cache'
+inputs:
+ go-version:
+ required: true
+runs:
+ using: "composite"
+ steps:
+ - name: Set up Go
+ uses: actions/setup-go@v2
+ with:
+ go-version: ${{ inputs.go-version }}
+ stable: ${{ !(contains(inputs.go-version, 'beta') || contains(inputs.go-version, 'rc')) }}
+
+ # Restore original modification time of files based on the date of the most
+ # recent commit that modified them as mtimes affect the Go test cache.
+ - name: Restore modification time of checkout files
+ shell: bash
+ run: |
+ # Set a base, fixed modification time of all directories.
+ # git-restore-mtime doesn't set the mtime of of all directories.
+ # (see https://github.com/MestreLion/git-tools/issues/47 for details)
+ touch -m -t '201509301646' $(find . -type d -not -path '.git/*')
+ # Restore original modification time from git. git clone sets the
+ # modification time to the current time, but Go tests that access fixtures
+ # get invalidated if their modification times change.
+ sudo apt-get install -y git-restore-mtime
+ git restore-mtime
+
+ # The PREFIX must uniquely identify the specific instance of a job executing.
+ - shell: bash
+ run: echo 'PREFIX=${{ github.workflow }}-${{ github.job }}-${{ runner.os }}-${{ inputs.go-version }}-matrix(${{ join(matrix.*,'|') }})' >> $GITHUB_ENV
+
+ # Cache the Go Modules downloaded during the job.
+ - uses: actions/cache@v2
+ with:
+ path: ~/go/pkg/mod
+ key: ${{ env.PREFIX }}-go-mod-${{ hashFiles('**/go.sum') }}
+ restore-keys: ${{ env.PREFIX }}-go-mod
+
+ # Cache any build and test artifacts during the job, which will speed up
+ # rebuilds and cause test runs to skip tests that have no reason to rerun.
+ # Don't run this for protected branches like master/main.
+ - uses: actions/cache@v2
+ if: ${{ github.ref_protected || github.ref != 'master' || github.ref != 'main' }}
+ with:
+ path: ~/.cache/go-build
+ key: ${{ env.PREFIX }}-go-build-${{ github.ref }}-${{ hashFiles('**', '!.git') }}
+ restore-keys: |
+ ${{ env.PREFIX }}-go-build-${{ github.ref }}
+ ${{ env.PREFIX }}-go-build

.github/workflows/go.yml

@@ -15,11 +15,9 @@ jobs:
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v2
- - name: Set up Go
- uses: actions/setup-go@v2
+ - uses: ./.github/actions/setup-go
  with:
  go-version: ${{ matrix.go }}
- stable: ${{ !contains(matrix.go, 'rc') }}
  - run: ./gomod.sh
  - run: ./gofmt.sh
  - run: ./gogenerate.sh
@@ -34,11 +32,9 @@ jobs:
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v2
- - name: Set up Go
- uses: actions/setup-go@v2
+ - uses: ./.github/actions/setup-go
  with:
  go-version: ${{ matrix.go }}
- stable: ${{ !contains(matrix.go, 'rc') }}
  - run: go build ./...
 
  test:
@@ -81,9 +77,11 @@ jobs:
  REDIS_PORT: 6379
  steps:
  - uses: actions/checkout@v2
- - name: Set up Go
- uses: actions/setup-go@v2
+ with:
+ # We need to full history for git-restore-mtime to know what modification dates to use.
+ # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
+ fetch-depth: '0'
+ - uses: ./.github/actions/setup-go
  with:
  go-version: ${{ matrix.go }}
- stable: ${{ !contains(matrix.go, 'rc') }}
  - run: go test -race -cover ./...

.github/workflows/horizon.yml

@@ -14,6 +14,7 @@ jobs:
  go: [1.17]
  pg: [9.6.5]
  ingestion-backend: [db, captive-core, captive-core-remote-storage]
+ captive-core: [18.0.3-746.f3baea6.focal]
  runs-on: ${{ matrix.os }}
  services:
  postgres:
@@ -38,20 +39,21 @@ jobs:
  PGDATABASE: postgres
  steps:
  - uses: actions/checkout@v2
- - name: Set up Go
- uses: actions/setup-go@v2
+ with:
+ # We need to full history for git-restore-mtime to know what modification dates to use.
+ # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
+ fetch-depth: '0'
+ - uses: ./.github/actions/setup-go
  with:
  go-version: ${{ matrix.go }}
  - name: Pull latest Stellar Core image
  run: docker pull stellar/stellar-core
  - if: ${{ startsWith(matrix.ingestion-backend, 'captive-core') }}
  name: Install and enable Captive Core
- env:
- CAPTIVE_CORE_VERSION: 18.0.3-746.f3baea6.focal
  run: |
  sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
  sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
- sudo apt-get update && sudo apt-get install -y stellar-core=$CAPTIVE_CORE_VERSION
+ sudo apt-get update && sudo apt-get install -y stellar-core=${{ matrix.captive-core }}
  echo "Using stellar core version $(stellar-core version)"
  echo 'HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true' >> $GITHUB_ENV
  echo 'CAPTIVE_CORE_BIN=/usr/bin/stellar-core' >> $GITHUB_ENV

services/horizon/internal/integration/db_test.go

@@ -3,6 +3,7 @@ package integration
 import (
  "context"
  "fmt"
+ "os"
  "path/filepath"
  "strconv"
  "testing"
@@ -538,6 +539,9 @@ func command(horizonConfig horizon.Config, args ...string) []string {
  // due to ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING
  "--checkpoint-frequency",
  "8",
+ // Create the storage directory outside of the source repo,
+ // otherwise it will break Golang test caching.
+ "--captive-core-storage-path=" + os.TempDir(),
  }, args...)
 }
 

services/horizon/internal/test/integration/integration.go

@@ -309,10 +309,13 @@ func (i *Test) StartHorizon() error {
  hostname,
  stellarCorePostgresPort,
  ),
- "stellar-core-binary-path": coreBinaryPath,
- "captive-core-config-path": captiveCoreConfigPath,
- "captive-core-http-port": "21626",
- "captive-core-use-db": captiveCoreUseDB,
+ "stellar-core-binary-path": coreBinaryPath,
+ "captive-core-config-path": captiveCoreConfigPath,
+ "captive-core-http-port": "21626",
+ "captive-core-use-db": captiveCoreUseDB,
+ // Create the storage directory outside of the source repo,
+ // otherwise it will break Golang test caching.
+ "captive-core-storage-path": os.TempDir(),
  "enable-captive-core-ingestion": strconv.FormatBool(len(coreBinaryPath) > 0),
  "ingest": "true",
  "history-archive-urls": fmt.Sprintf("http:https://%s:%d", hostname, historyArchivePort),