ci: add integration test job for modflow/exes

.github/workflows/integration.yml

@@ -0,0 +1,121 @@
+name: Integration testing
+on:
+ push:
+ branches:
+ - main
+ - develop*
+ - ci-diagnose*
+ paths-ignore:
+ - '**.md'
+ pull_request:
+ branches:
+ - main
+ - develop
+ paths-ignore:
+ - '**.md'
+ schedule:
+ - cron: '0 6 * * *' # run at 6 AM UTC every day
+jobs:
+ test_modflow:
+ name: MODFLOW 6 integration tests
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ ubuntu-20.04, macos-latest, windows-2019 ]
+ defaults:
+ run:
+ shell: bash -l {0}
+ steps:
+
+ - name: Checkout action
+ uses: actions/checkout@v3
+ with:
+ path: executables
+
+ - name: Checkout modflow6
+ uses: actions/checkout@v3
+ with:
+ repository: MODFLOW-USGS/modflow6
+ path: modflow6
+
+ - name: Setup Micromamba
+ uses: mamba-org/provision-with-micromamba@main
+ with:
+ environment-file: modflow6/environment.yml
+
+ - name: Setup ifort
+ uses: modflowpy/install-intelfortran-action@v1
+ # with:
+ # path: ${{ runner.os != 'Windows' && 'bin' || 'C:\Program Files (x86)\Intel\oneAPI' }}
+
+ - name: Fix Micromamba path (Windows)
+ if: runner.os == 'Windows'
+ shell: pwsh
+ run: |
+ # https://github.com/modflowpy/install-intelfortran-action#conda-scripts
+ $mamba_bin = "C:\Users\runneradmin\micromamba-root\envs\modflow6\Scripts"
+ echo $mamba_bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+ - name: Build modflow6 (Linux & Mac)
+ working-directory: modflow6
+ run: |
+ meson setup builddir -Ddebug=false --prefix=$(pwd) --libdir=bin
+ meson compile -v -C builddir
+ meson install -C builddir
+
+ # - name: Show meson build log
+ # run: cat modflow6/builddir/meson-logs/meson-log.txt
+
+ - name: Update FloPy
+ working-directory: modflow6/autotest
+ run: python update_flopy.py
+
+ - name: Get executables
+ if: runner.os != 'Windows'
+ working-directory: modflow6/autotest
+ run: |
+ pytest -v -s get_exes.py
+ sudo rm -rf ../bin/downloaded/*
+
+ - name: Get executables
+ if: runner.os == 'Windows'
+ shell: pwsh
+ working-directory: modflow6/autotest
+ run: |
+ pytest -v -s get_exes.py
+ rm -Force ../bin/downloaded/*
+
+ - name: Show pymake version
+ run: |
+ pip show mfpymake
+ python -c "import pymake; print(pymake.__version__)"
+
+ - name: Build executables
+ if: runner.os != 'Windows'
+ working-directory: executables/scripts
+ run: python build_executables.py -p ../../modflow6/bin/downloaded
+
+ - name: Build executables
+ if: runner.os == 'Windows'
+ shell: cmd
+ working-directory: executables/scripts
+ run: python build_executables.py -p ../../modflow6/bin/downloaded
+
+ # - name: Build executables
+ # uses: nick-fields/retry@v2
+ # with:
+ # timeout_minutes: 40
+ # max_attempts: 5
+ # command: |
+ # cd modflow6/bin/downloaded
+ # python ../../../scripts/build_executables.py
+
+ - name: Set executable permission
+ if: runner.os != 'Windows'
+ working-directory: modflow6/bin/downloaded
+ run: sudo chmod +x *
+
+ - name: Test modflow6
+ working-directory: modflow6/autotest
+ run: pytest -v -n auto --durations 0

.github/workflows/continuous_integration.yml → .github/workflows/release.yml

@@ -1,11 +1,9 @@
-name: executables continuous integration
-
+name: Release MODFLOW executables
 on:
  schedule:
  - cron: '0 3 * * 3' # run at 3 AM UTC every Wednesday
  push:
  branches: [ master ]
- pull_request:
  workflow_dispatch:
 jobs:
  executables-intel:
@@ -51,21 +49,18 @@ jobs:
 
  - name: build executables on Linux and macOS
  if: runner.os != 'Windows'
- run: |
- python .github/common/build_executables.py
+ run: python scripts/build_executables.py
 
  - name: build executables on Windows
  if: runner.os == 'Windows'
  shell: cmd
- run: |
- python .github/common/build_executables.py
+ run: python scripts/build_executables.py
 
  - name: Upload a Build Artifact
  uses: actions/[email protected]
  with:
  name: release_build
- path: |
- ./*.zip
+ path: ./*.zip
 
  - name: Upload additional Build Artifacts
  if: runner.os == 'Linux'
@@ -147,11 +142,12 @@ jobs:
  - name: Update readme
  id: update-readme
  run: |
+ # update readme from metadata
  cp release_build/code.md code.md
  cp release_build/code.json code.json
  python update_readme.py
- stat README.md
- cat README.md
+ 
+ # determine whether changes need to be committed
  readme_changed=$(git diff --exit-code README.md)
  if [ "$readme_changed" -eq "1" ]; then
  echo "Changes to README.md:"

DEVELOPER.md

@@ -18,7 +18,7 @@ This repository only builds USGS programs and contains none of their source code
 
 As such, this repo's CI is configured to allow manually triggering releases, independent of changes to version-controlled files.
 
-The `.github/workflows/continuous_integration.yml` CI workflow is triggered on the following events:
+The `.github/workflows/release.yml` CI workflow is triggered on the following events:
 
 - `push` to `master`
 - `pull_request` to any branch
@@ -41,17 +41,17 @@ Navigate to the Actions tab of this repository. Select the `executables continuo
 Install and configure the [GitHub CLI](https://cli.github.com/manual/) if needed. Then the following command can be run from the root of your local clone of the repository:
 
 ```shell
-gh workflow run continuous_integration.yml
+gh workflow run release.yml
 ```
 
 On the first run, the CLI will prompt to choose whether the run should be triggered on your fork of the repository or on the upstream version. This decision is stored for subsequent runs — to override it later, use the `--repo` (short `-R`) option to specify the repository. For instance, if you initially selected your fork but would like to trigger a release on the main repository:
 
 ```shell
-gh workflow run continuous_integration.yml -R MODFLOW-USGS/executables
+gh workflow run release.yml -R MODFLOW-USGS/executables
 ```
 
 **Note:** by default, workflow runs are associated with the repository's default branch. If the repo's default branch is `develop` (as is currently the case for `MODFLOW-USGS/executables`, you will need to use the `--ref` (short `-r`) option to specify the `master` branch when triggering a release from the CLI. For instance:
 
 ```shell
-gh workflow run continuous_integration.yml -R MODFLOW-USGS/executables -r master
+gh workflow run release.yml -R MODFLOW-USGS/executables -r master
 ```

scripts/build_executables.py

@@ -0,0 +1,100 @@
+import argparse
+import sys
+import subprocess
+import textwrap
+from pathlib import Path
+
+DEFAULT_RETRIES = 3
+
+
+def get_ostag() -> str:
+ """Determine operating system tag from sys.platform."""
+ if sys.platform.startswith("linux"):
+ return "linux"
+ elif sys.platform.startswith("win"):
+ return "win64"
+ elif sys.platform.startswith("darwin"):
+ return "mac"
+ raise ValueError(f"platform {sys.platform!r} not supported")
+
+
+def get_cc() -> str:
+ """Determine operating system tag from sys.platform."""
+ if sys.platform.startswith("linux"):
+ return "icc"
+ elif sys.platform.startswith("win"):
+ return "icl"
+ elif sys.platform.startswith("darwin"):
+ return "icc"
+ raise ValueError(f"platform {sys.platform!r} not supported")
+
+
+def mfpymake_run_command(args) -> bool:
+ success = False
+ for idx in range(DEFAULT_RETRIES):
+ p = subprocess.run(args)
+ if p.returncode == 0:
+ success = True
+ break
+ print(f"{args[0]} run {idx + 1}/{DEFAULT_RETRIES} failed...rerunning")
+ return success
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ prog=f"Build MODFLOW-related binaries and metadata files",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=textwrap.dedent(
+ """\
+ Build MODFLOW-releated executables, shared libraries and metadata files with pymake.
+ """
+ ),
+ )
+ parser.add_argument(
+ "-p",
+ "--path",
+ required=False,
+ default=get_ostag(),
+ help="Path to create built binaries and metadata files",
+ )
+ parser.add_argument(
+ "-r",
+ "--retries",
+ type=int,
+ required=False,
+ default=DEFAULT_RETRIES,
+ help="Number of times to retry a failed build",
+
+ )
+ args = parser.parse_args()
+
+ # output path
+ path = Path(args.path)
+ path.mkdir(parents=True, exist_ok=True)
+
+ # number of retries
+ retries = args.retries
+
+ # C compiler
+ cc = get_cc()
+
+ cmd = [
+ "make-code-json",
+ "-f",
+ str(path / "code.json"),
+ "--verbose",
+ ]
+
+ if not mfpymake_run_command(cmd):
+ raise RuntimeError(f"could not run {cmd[0]}")
+
+ cmd = [
+ "make-program", ":",
+ f"--appdir={path}",
+ "-fc=ifort", f"-cc={cc}",
+ f"--zip={path}.zip",
+ "--keep",
+ ]
+
+ if not mfpymake_run_command(cmd):
+ raise RuntimeError("could not build the executables")

update_readme.py → scripts/update_readme.py

@@ -6,26 +6,12 @@
 FILES = ("code.json", "code.md")
 
 
-def _create_code_json() -> None:
- subprocess.run(
- [
- "make-code-json",
- "-f",
- f"code.json",
- "--verbose",
- ]
- )
-
- if not target_file.is_file():
- raise FileNotFoundError(f"{target_file} does not exist")
-
-
 def _update_readme() -> None:
- with open("README.md", "r") as f:
+ with open("../README.md", "r") as f:
  readme_md = f.read().splitlines()
  with open("code.md", "r") as f:
  code_md = f.read().splitlines()
- with open("README.md", "w") as f:
+ with open("../README.md", "w") as f:
  for line in readme_md:
  if TAG not in line:
  f.write(f"{line}\n")