feat(ci): Bazel remote cache on s3 (#369)

* feat(ci): Bazel remote cache on s3

* fix

.bazelrc

@@ -36,9 +36,15 @@ build --@rules_rust//:extra_rustc_flag=-Dbindings_with_variant_name
 build --strip=never
 
 # build:dfinity --remote_cache=bazel-remote.idx.dfinity.network
-build --experimental_remote_cache_async
+build --remote_cache=grpc:https://localhost:9092
+build --remote_instance_name=default
+build --google_default_credentials=false
+build --remote_upload_local_results=false
+# build --experimental_remote_cache_async
+build --experimental_remote_cache_compression # compress/decompress cache blobs with zstd
 build --remote_timeout=30s # Default is 60s.
 build:ci --remote_timeout=5m # Default is 60s.
+build:ci --remote_upload_local_results=true
 
 # build:dfinity --experimental_remote_downloader=bazel-remote.idx.dfinity.network --experimental_remote_downloader_local_fallback
 build:local --experimental_remote_downloader=

.github/workflows/main.yaml

@@ -26,6 +26,21 @@ jobs:
  - uses: actions/checkout@v4
  with:
  fetch-depth: 2
+ - name: "☁️ Setup S3 cache credentials"
+ id: bazel-remote-creds
+ shell: bash
+ run: |
+ mkdir -p ~/.aws
+ cat <<_EOF | install -m 0600 /dev/stdin ~/.aws/config
+ [wasabi]
+ region = eu-central-2
+ _EOF
+
+ cat <<_EOF | install -m 0600 /dev/stdin ~/.aws/credentials
+ [wasabi]
+ aws_access_key_id = ${{ secrets.WASABI_ACCESS_KEY }}
+ aws_secret_access_key = ${{ secrets.WASABI_SECRET_KEY }}
+ _EOF
  - name: "☁️ Setup runner"
  uses: ./.github/workflows/manage-runner-pre
 

.github/workflows/manage-runner-post/action.yaml

@@ -9,6 +9,13 @@ inputs:
 runs:
  using: composite
  steps:
+ ########################################
+ # Stop bazel-remote cache
+ ########################################
+ - name: "💾 Stop bazel-remote s3 cache"
+ run: ./bin/bazel-cache-s3-stop.sh
+ shell: bash
+
  ########################################
  # Optimize bazel cache by hard-linking duplicate files
  ########################################

.github/workflows/manage-runner-pre/action.yaml

@@ -35,3 +35,7 @@ runs:
  # Bazel actually keeps all intermediate objects so builds are still fast
  run: bazel clean
  shell: bash
+
+ - name: "💾 Start bazel-remote s3 cache"
+ run: ./bin/bazel-cache-s3-start.sh
+ shell: bash

bin/bazel-cache-s3-start.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eExuo pipefail
+
+mkdir -p ~/.cache/bazel-remote
+docker run --detach --name=bazel-remote -u $(id -u):$(id -g) \
+ -v ~/.cache/bazel-remote:/data -v $HOME/.aws:/aws-config \
+ -p 9090:8080 -p 9092:9092 \
+ quay.io/bazel-remote/bazel-remote \
+ --max_size 20 \
+ --s3.auth_method=aws_credentials_file --s3.aws_profile=wasabi --s3.aws_shared_credentials_file=/aws-config/credentials --s3.bucket=dre-ci-cache --s3.endpoint=s3.eu-central-2.wasabisys.com

bin/bazel-cache-s3-stop.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -eExuo pipefail
+
+docker logs bazel-remote
+docker stop bazel-remote
+docker rm bazel-remote