split saving the compilation cache into a shared workflow

.github/actions/cache-query-compilation/action.yml

@@ -0,0 +1,60 @@
+name: Cache query compilation
+description: Caches CodeQL compilation caches - should be run both on PRs and pushes to main.
+
+inputs:
+ key:
+ description: 'The cache key to use - should be unique to the workflow'
+ required: true
+
+outputs:
+ cache-dir:
+ description: "The directory where the cache was stored"
+ value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+
+runs:
+ using: composite
+ steps:
+ # Cache the query compilation caches.
+ # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
+ - name: Calculate merge-base
+ shell: bash
+ if: ${{ github.event_name == 'pull_request' }}
+ env:
+ BASE_BRANCH: ${{ github.base_ref }}
+ run: |
+ MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
+ echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
+ - name: Read CodeQL query compilation - PR
+ if: ${{ github.event_name == 'pull_request' }}
+ uses: actions/cache@v3
+ with:
+ path: '**/.cache'
+ key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
+ restore-keys: |
+ codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge-base }}
+ codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-
+ codeql-compile-${{ inputs.key }}-main-
+ - name: Fill CodeQL query compilation cache - main
+ if: ${{ github.event_name != 'pull_request' }}
+ uses: actions/cache@v3
+ with:
+ path: '**/.cache'
+ key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+ restore-keys: | # restore from another random commit, to speed up compilation.
+ codeql-compile-${{ inputs.key }}-${{ github.ref_name }}- 
+ codeql-compile-${{ inputs.key }}-main-
+ - name: Fill compilation cache directory
+ id: fill-compilation-dir
+ shell: bash 
+ run: |
+ # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+ mkdir -p ${COMBINED_CACHE_DIR}
+ rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+ # copy the contents of the .cache folders into the combined cache folder.
+ cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+ # clean up the .cache folders
+ rm -rf **/.cache/*
+
+ echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
+ env: 
+ COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir

.github/workflows/compile-queries.yml

@@ -14,58 +14,26 @@ jobs:
 
  steps:
  - uses: actions/checkout@v3
- # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
- - name: Calculate merge-base
- if: ${{ github.event_name == 'pull_request' }}
- env:
- BASE_BRANCH: ${{ github.base_ref }}
- run: |
- MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
- echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
- - name: Read CodeQL query compilation - PR
- if: ${{ github.event_name == 'pull_request' }}
- uses: actions/cache@v3
- with:
- path: '**/.cache'
- key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
- restore-keys: |
- codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
- codeql-compile-${{ github.base_ref }}-
- codeql-compile-main-
- - name: Fill CodeQL query compilation cache - main
- if: ${{ github.event_name != 'pull_request' }}
- uses: actions/cache@v3
- with:
- path: '**/.cache'
- key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
- restore-keys: | # restore from another random commit, to speed up compilation.
- codeql-compile-${{ github.ref_name }}- 
- codeql-compile-main-
  - name: Setup CodeQL
  uses: ./.github/actions/fetch-codeql
  with:
  channel: 'release'
+ - name: Cache compilation cache
+ id: query-cache
+ uses: ./.github/actions/cache-query-compilation
+ with: 
+ key: all-queries
  - name: check formatting
  run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
  - name: compile queries - check-only
  # run with --check-only if running in a PR (github.sha != main)
  if : ${{ github.event_name == 'pull_request' }}
  shell: bash
- run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only
+ run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
  - name: compile queries - full
  # do full compile if running on main - this populates the cache
  if : ${{ github.event_name != 'pull_request' }}
  shell: bash
- run: |
- # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
- mkdir -p ${COMBINED_CACHE_DIR}
- rm -f */ql/{src,examples}/.cache/{lock,size} # -f to avoid errors if the cache is empty.
- # copy the contents of the .cache folders into the combined cache folder.
- cp -r */ql/{src,examples}/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
- # clean up the .cache folders
- rm -rf */ql/{src,examples}/.cache/*
-
- # compile the queries
- codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache ${COMBINED_CACHE_DIR}
+ run: codeql query compile -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
  env: 
  COMBINED_CACHE_DIR: ${{ github.workspace }}/compilation-dir