CI: Cache GSHHG and DCW data as artifacts to simplify workflows (#8481)

GenericMappingTools · May 31, 2024 · 58b5bea · 58b5bea
1 parent 5e53ea3
commit 58b5bea
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 83 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -71,7 +71,7 @@ jobs:
  env:
  GH_TOKEN: ${{ github.token }}
 
- - name: Setup conda
+ - name: Setup conda (macOS)
  uses: conda-incubator/setup-miniconda@v3
  if: runner.os == 'macOS'
 
@@ -84,16 +84,10 @@ jobs:
  env:
  EXCLUDE_OPTIONAL: ${{ matrix.EXCLUDE_OPTIONAL }}
 
- - name: Cache GSHHG and DCW data
- uses: actions/cache@v4
- id: cache-coastline
- with:
- path: ${{ env.COASTLINEDIR }}
- key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
-
- - name: Download coastlines
- run: bash ci/download-coastlines.sh
- if: steps.cache-coastline.outputs.cache-hit != 'true'
+ - name: Download GSHHG and DCW data
+ run: gh run download -n coastline-cache -D coastline
+ env:
+ GH_TOKEN: ${{ github.token }}
 
  - name: Configure GMT
  run: |
@@ -121,20 +115,18 @@ jobs:
  cmake --build .
  if: runner.os == 'Windows'
 
- - name: Download cached GMT remote data from GitHub Artifacts
- run: |
- gh run download -n gmt-cache -D ~/.gmt/static/
- ls -lRh ~/.gmt/static/
- env:
- GH_TOKEN: ${{ github.token }}
-
  - name: Install GMT
  run: |
  cd build
  cmake --build . --target install
  # Add GMT PATH to bin
  echo "${INSTALLDIR}/bin" >> $GITHUB_PATH
 
+ - name: Download cached GMT remote data from GitHub Artifacts
+ run: gh run download -n gmt-cache -D ~/.gmt/static/
+ env:
+ GH_TOKEN: ${{ github.token }}
+
  - name: Check a few simple commands
  run: bash ci/simple-gmt-tests.sh
 

diff --git a/.github/workflows/ci-caches.yml b/.github/workflows/ci-caches.yml
@@ -2,7 +2,8 @@
 # Create CI caches for:
 #
 # 1. GMT remote data for building documentation and running tests
-# 2. vcpkg libraries on Windows
+# 2. GMT GSHHG and DCW datasets
+# 3. vcpkg libraries on Windows
 #
 name: GMT CI Caches
 
@@ -105,8 +106,28 @@ jobs:
  uses: actions/upload-artifact@v4
  with:
  name: gmt-cache
- path: |
- ~/.gmt/static
+ path: ~/.gmt/static
+
+ coastline_cache:
+ name: Cache GSHHG and DCW datasets
+ runs-on: ubuntu-latest
+ env:
+ COASTLINEDIR: ${{ github.workspace }}/coastline
+
+ steps:
+ - name: Checkout
+ uses: actions/[email protected]
+
+ - name: Download GSHHG and DCW datasets
+ run: bash ci/download-coastlines.sh
+ env:
+ COASTLINEDIR: ${{ env.COASTLINEDIR }}
+
+ - name: Upload artifacts to GitHub
+ uses: actions/upload-artifact@v4
+ with:
+ name: coastline-cache
+ path: ${{ env.COASTLINEDIR }}
 
  vcpkg_cache:
  name: Cache vcpkg libraries

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -67,9 +67,6 @@ jobs:
  - fedora:rawhide # rolling release with latest versions
 
  steps:
- - name: Checkout
- uses: actions/[email protected]
-
  - name: Install GMT dependencies
  run: |
  os=$(cat /etc/os-release | grep "^ID=" | awk -F= '{print $2}')
@@ -83,24 +80,29 @@ jobs:
  ghostscript curl git
  apt reinstall -y ca-certificates
  update-ca-certificates
+ # Install gh from binary deb package
+ curl -LO https://github.com/cli/cli/releases/download/v2.49.0/gh_2.49.0_linux_amd64.deb
+ apt install ./gh_2.49.0_linux_amd64.deb
  elif [[ "$os" = "fedora" ]]; then
  dnf install -y \
  cmake ninja-build \
  libcurl-devel netcdf-devel gdal-devel gdal \
  fftw3-devel pcre-devel lapack-devel openblas-devel glib2-devel \
- ghostscript openssl
+ ghostscript openssl gh
  fi
 
- - name: Cache GSHHG and DCW data
- uses: actions/cache@v4
- id: cache-coastline
- with:
- path: ${{ env.COASTLINEDIR }}
- key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
+ - name: Checkout
+ uses: actions/[email protected]
+
+ # Fix the git error:
+ # "failed to run git: fatal: detected dubious ownership in repository at '/__w/gmt/gmt'"
+ - name: Fixup git
+ run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
- - name: Download coastlines
- run: bash ci/download-coastlines.sh
- if: steps.cache-coastline.outputs.cache-hit != 'true'
+ - name: Download GSHHG and DCW data
+ run: gh run download -n coastline-cache -D coastline
+ env:
+ GH_TOKEN: ${{ github.token }}
 
  - name: Configure GMT
  run: bash ci/config-gmt-unix.sh
@@ -116,18 +118,9 @@ jobs:
  echo "${INSTALLDIR}/bin" >> $GITHUB_PATH
 
  - name: Download cached GMT remote data from GitHub Artifacts
- uses: dawidd6/[email protected]
- with:
- workflow: ci-caches.yml
- name: gmt-cache
- path: gmt-cache
-
- # Move downloaded files to ~/.gmt directory and list them
- - name: Move and list downloaded remote files
- run: |
- mkdir -p ~/.gmt/static/
- mv gmt-cache/* ~/.gmt/static/
- ls -lRh ~/.gmt/static/
+ run: gh run download -n gmt-cache -D ~/.gmt/static/
+ env:
+ GH_TOKEN: ${{ github.token }}
 
  - name: Check a few simple commands
  run: bash ci/simple-gmt-tests.sh
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -65,7 +65,7 @@ jobs:
  env:
  GH_TOKEN: ${{ github.token }}
 
- - name: Setup conda
+ - name: Setup conda (macOS)
  uses: conda-incubator/setup-miniconda@v3
  if: runner.os == 'macOS'
 
@@ -76,19 +76,10 @@ jobs:
  os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
  bash ci/install-dependencies-${os}.sh
 
- - name: Cache GSHHG and DCW data
- uses: actions/cache@v4
- id: cache-coastline
- with:
- path: ${{ env.COASTLINEDIR }}
- key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
-
- - name: Download coastlines
- run: bash ci/download-coastlines.sh
- if: steps.cache-coastline.outputs.cache-hit != 'true'
-
- - name: Pull baseline image data from dvc remote
- run: dvc pull --no-run-cache
+ - name: Download GSHHG and DCW data
+ run: gh run download -n coastline-cache -D coastline
+ env:
+ GH_TOKEN: ${{ github.token }}
 
  - name: Configure GMT
  run: |
@@ -116,12 +107,8 @@ jobs:
  cmake --build .
  if: runner.os == 'Windows'
 
- - name: Download cached GMT remote data from GitHub Artifacts
- run: |
- gh run download -n gmt-cache -D ~/.gmt/static/
- ls -lRh ~/.gmt/static/
- env:
- GH_TOKEN: ${{ github.token }}
+ - name: Pull baseline image data from dvc remote
+ run: dvc pull --no-run-cache
 
  - name: Build documentation
  run: |

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -90,16 +90,10 @@ jobs:
  New-ItemProperty -Path "HKLM:\Software\GPL Ghostscript\10.02" -Name GS_LIB -PropertyType String -Value "C:\Miniconda\Library\bin;C:\Miniconda\Library\lib;C\Miniconda\Library\Font;C:\Miniconda\Library\fonts"
  if: runner.os == 'Windows'
 
- - name: Cache GSHHG and DCW data
- uses: actions/cache@v4
- id: cache-coastline
- with:
- path: ${{ env.COASTLINEDIR }}
- key: coastline-${{ hashFiles('ci/download-coastlines.sh') }}
-
- - name: Download coastlines
- run: bash ci/download-coastlines.sh
- if: steps.cache-coastline.outputs.cache-hit != 'true'
+ - name: Download GSHHG and DCW data
+ run: gh run download -n coastline-cache -D coastline
+ env:
+ GH_TOKEN: ${{ github.token }}
 
  - name: Configure GMT
  run: |
@@ -127,13 +121,6 @@ jobs:
  cmake --build .
  if: runner.os == 'Windows'
 
- - name: Download cached GMT remote data from GitHub Artifacts
- run: |
- gh run download -n gmt-cache -D ~/.gmt/static/
- ls -lRh ~/.gmt/static/
- env:
- GH_TOKEN: ${{ github.token }}
-
  - name: Pull baseline image data from dvc remote
  run: dvc pull --no-run-cache
 
@@ -144,6 +131,11 @@ jobs:
  # Add GMT PATH to bin
  echo "${INSTALLDIR}/bin" >> $GITHUB_PATH
 
+ - name: Download cached GMT remote data from GitHub Artifacts
+ run: gh run download -n gmt-cache -D ~/.gmt/static/
+ env:
+ GH_TOKEN: ${{ github.token }}
+
  - name: Check a few simple commands
  run: bash ci/simple-gmt-tests.sh