From 56536499a1878001e16f591e4e4d18ed08ae7e11 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Mon, 27 Feb 2023 09:13:57 -0500 Subject: [PATCH] ci: refactor workflows, add tests, bump action versions, update docs (#17) * refactor workflows * create code.json in build_executables.py * move scripts from .github/common to /scripts * add job for modflow6 integration testing * use GITHUB_TOKEN to avoid rate limits * update paths to use pathlib * unpin/upgrade actions * update dev docs --- .github/common/build_executables.py | 63 --------- .github/workflows/integration.yml | 125 ++++++++++++++++++ ...continuous_integration.yml => release.yml} | 43 +++--- DEVELOPER.md | 27 ++-- scripts/build_executables.py | 98 ++++++++++++++ update_readme.py => scripts/update_readme.py | 27 +--- 6 files changed, 264 insertions(+), 119 deletions(-) delete mode 100755 .github/common/build_executables.py create mode 100644 .github/workflows/integration.yml rename .github/workflows/{continuous_integration.yml => release.yml} (88%) create mode 100755 scripts/build_executables.py rename update_readme.py => scripts/update_readme.py (52%) diff --git a/.github/common/build_executables.py b/.github/common/build_executables.py deleted file mode 100755 index c181833..0000000 --- a/.github/common/build_executables.py +++ /dev/null @@ -1,63 +0,0 @@ -import sys -import subprocess - -RETRIES = 3 - - -def get_ostag() -> str: - """Determine operating system tag from sys.platform.""" - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform.startswith("win"): - return "win64" - elif sys.platform.startswith("darwin"): - return "mac" - raise ValueError(f"platform {sys.platform!r} not supported") - - -def get_cctag() -> str: - """Determine operating system tag from sys.platform.""" - if sys.platform.startswith("linux"): - return "icc" - elif sys.platform.startswith("win"): - return "icl" - elif sys.platform.startswith("darwin"): - return "icc" - raise ValueError(f"platform {sys.platform!r} not supported") - - -def mfpymake_run_command(args) -> bool: - success = False - for idx in range(RETRIES): - p = subprocess.run(args) - if p.returncode == 0: - success = True - break - print(f"{args[0]} run {idx + 1}/{RETRIES} failed...rerunning") - return success - - -if __name__ == "__main__": - cmd = [ - "make-code-json", - "-f", - f"{get_ostag()}/code.json", - "--verbose", - ] - - if not mfpymake_run_command(cmd): - raise RuntimeError(f"could not run {cmd[0]}") - - cmd = [ - "make-program", - ":", - f"--appdir={get_ostag()}", - "-fc=ifort", - f"-cc={get_cctag()}", - f"--zip={get_ostag()}.zip", - "--keep", - ] - - if not mfpymake_run_command(cmd): - raise RuntimeError("could not build the executables") - diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..27b94a2 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,125 @@ +name: Integration testing +on: + push: + branches: + - main + - develop* + - ci-diagnose* + paths-ignore: + - '**.md' + pull_request: + branches: + - main + - develop + paths-ignore: + - '**.md' + schedule: + - cron: '0 6 * * *' # run at 6 AM UTC every day +jobs: + test_modflow: + name: MODFLOW 6 integration tests + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-20.04, macos-latest, windows-2019 ] + defaults: + run: + shell: bash -l {0} + steps: + + - name: Checkout action + uses: actions/checkout@v3 + with: + path: executables + + - name: Checkout modflow6 + uses: actions/checkout@v3 + with: + repository: MODFLOW-USGS/modflow6 + path: modflow6 + + - name: Setup Micromamba + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: modflow6/environment.yml + + - name: Setup ifort + uses: modflowpy/install-intelfortran-action@v1 + # with: + # path: ${{ runner.os != 'Windows' && 'bin' || 'C:\Program Files (x86)\Intel\oneAPI' }} + + - name: Fix Micromamba path (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + # https://github.com/modflowpy/install-intelfortran-action#conda-scripts + $mamba_bin = "C:\Users\runneradmin\micromamba-root\envs\modflow6\Scripts" + echo $mamba_bin | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Build modflow6 (Linux & Mac) + working-directory: modflow6 + run: | + meson setup builddir -Ddebug=false --prefix=$(pwd) --libdir=bin + meson compile -v -C builddir + meson install -C builddir + + # - name: Show meson build log + # run: cat modflow6/builddir/meson-logs/meson-log.txt + + - name: Update FloPy + working-directory: modflow6/autotest + run: python update_flopy.py + + - name: Get executables + if: runner.os != 'Windows' + working-directory: modflow6/autotest + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + pytest -v -s get_exes.py + sudo rm -rf ../bin/downloaded/* + + - name: Get executables + if: runner.os == 'Windows' + shell: pwsh + working-directory: modflow6/autotest + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + pytest -v -s get_exes.py + rm -Force ../bin/downloaded/* + + - name: Show pymake version + run: | + pip show mfpymake + python -c "import pymake; print(pymake.__version__)" + + - name: Build executables + if: runner.os != 'Windows' + working-directory: executables/scripts + run: python build_executables.py -p ../../modflow6/bin/downloaded + + - name: Build executables + if: runner.os == 'Windows' + shell: cmd + working-directory: executables/scripts + run: python build_executables.py -p ../../modflow6/bin/downloaded + + # - name: Build executables + # uses: nick-fields/retry@v2 + # with: + # timeout_minutes: 40 + # max_attempts: 5 + # command: | + # cd modflow6/bin/downloaded + # python ../../../scripts/build_executables.py + + - name: Set executable permission + if: runner.os != 'Windows' + working-directory: modflow6/bin/downloaded + run: sudo chmod +x * + + - name: Test modflow6 + working-directory: modflow6/autotest + run: pytest -v -n auto --durations 0 diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/release.yml similarity index 88% rename from .github/workflows/continuous_integration.yml rename to .github/workflows/release.yml index 0f79edf..89141f5 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,11 @@ -name: executables continuous integration - +name: Release MODFLOW executables on: schedule: - cron: '0 3 * * 3' # run at 3 AM UTC every Wednesday push: - branches: [ master ] - pull_request: + branches: + - master + - develop workflow_dispatch: jobs: executables-intel: @@ -50,26 +50,23 @@ jobs: pip list - name: build executables on Linux and macOS - if: runner.os != 'Windows' - run: | - python .github/common/build_executables.py + # if: runner.os != 'Windows' + run: python scripts/build_executables.py - - name: build executables on Windows - if: runner.os == 'Windows' - shell: cmd - run: | - python .github/common/build_executables.py + # - name: build executables on Windows + # if: runner.os == 'Windows' + # shell: cmd + # run: python scripts/build_executables.py - name: Upload a Build Artifact - uses: actions/upload-artifact@v3.1.1 + uses: actions/upload-artifact@v3 with: name: release_build - path: | - ./*.zip + path: ./*.zip - name: Upload additional Build Artifacts if: runner.os == 'Linux' - uses: actions/upload-artifact@v3.1.0 + uses: actions/upload-artifact@v3 with: name: release_build path: | @@ -119,7 +116,7 @@ jobs: echo "$repo next version is $next" - name: Download release artifact - uses: actions/download-artifact@v3.0.0 + uses: actions/download-artifact@v3 with: name: release_build path: ./release_build/ @@ -140,20 +137,20 @@ jobs: - name: Build release body run: | - cat Header.md cat Header.md ./release_build/code.md > BodyFile.md cat BodyFile.md - name: Update readme id: update-readme run: | + # update readme from metadata cp release_build/code.md code.md cp release_build/code.json code.json - python update_readme.py - stat README.md - cat README.md + python scripts/update_readme.py + + # determine whether changes need to be committed readme_changed=$(git diff --exit-code README.md) - if [ "$readme_changed" -eq "1" ]; then + if [ "$readme_changed" -eq 1 ]; then echo "Changes to README.md:" git diff README.md changes="true" @@ -191,7 +188,7 @@ jobs: - name: Create release # only create new release on manual trigger if: github.event_name == 'workflow_dispatch' - uses: ncipollo/release-action@v1.11.2 + uses: ncipollo/release-action@v1 with: tag: ${{ env.NEXT_VERSION }} name: "MODFLOW and related programs binary executables" diff --git a/DEVELOPER.md b/DEVELOPER.md index 4f7c270..9ae6e17 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -1,10 +1,11 @@ # Developers -This document provides guidance for developers to use this repository to release USGS executables. +This document provides guidance for using this repository to release USGS executables. + - [Overview](#overview) - [Triggering a release](#triggering-a-release) - [GitHub UI](#github-ui) @@ -16,17 +17,25 @@ This document provides guidance for developers to use this repository to release This repository only builds USGS programs and contains none of their source code. Its contents are concerned only with the build and release process. Repository contents have no direct relationship to release cadence or version/tag numbers. -As such, this repo's CI is configured to allow manually triggering releases, independent of changes to version-controlled files. +This repo is configured to allow manually triggering releases, independent of changes to version-controlled files. -The `.github/workflows/continuous_integration.yml` CI workflow is triggered on the following events: +The `.github/workflows/release.yml` workflow is triggered on the following [events](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows): - `push` to `master` - `pull_request` to any branch - `workflow_dispatch` -If the triggering event is a `push` or `pull_request`, binaries are built and uploaded as artifacts, then the workflow ends. If the triggering event is a `workflow_dispatch`, binaries are built and uploaded as artifacts, then a release is created, incrementing the version number. The release is *not* a draft, and is published immediately. If changes are detected to any of the program versions or timestamps, a draft PR is opened against the `master` branch to update the table in `README.md`. +If the triggering event is `push` or `pull_request`, metadata is updated, binaries are built and uploaded as artifacts, and the workflow ends. + +If the triggering event is a `workflow_dispatch`: +- Metadata is updated. If changes are detected to any of the program versions or timestamps, a draft PR is opened against the `master` branch to update the table in `README.md`. +- Binaries are built and uploaded as artifacts +- A release is created, incrementing the version number. + + +**Note**: the release is currently published immediately, but could be changed to a draft by updating the `draft` input on `ncipollo/release-action` in `.github/workflows/release.yml`. -**Note:** version numbers do not currently follow semantic versioning conventions. We simply increment an integer version number by 1 for each release. +**Note**: version numbers don't currently follow semantic versioning conventions, but simply increment an integer for each release. ## Triggering a release @@ -34,24 +43,24 @@ The `workflow_dispatch` event is GitHub's mechanism for manually triggering work ### GitHub UI -Navigate to the Actions tab of this repository. Select the `executables continuous integration` workflow. A `Run workflow` button should be visible in an alert at the top of the list of workflow runs. Click the `Run workflow` button, selecting the `master` branch. +Navigate to the Actions tab of this repository. Select the release workflow. A `Run workflow` button should be visible in an alert at the top of the list of workflow runs. Click the `Run workflow` button, selecting the `master` branch. ### GitHub CLI Install and configure the [GitHub CLI](https://cli.github.com/manual/) if needed. Then the following command can be run from the root of your local clone of the repository: ```shell -gh workflow run continuous_integration.yml +gh workflow run release.yml ``` On the first run, the CLI will prompt to choose whether the run should be triggered on your fork of the repository or on the upstream version. This decision is stored for subsequent runs — to override it later, use the `--repo` (short `-R`) option to specify the repository. For instance, if you initially selected your fork but would like to trigger a release on the main repository: ```shell -gh workflow run continuous_integration.yml -R MODFLOW-USGS/executables +gh workflow run release.yml -R MODFLOW-USGS/executables ``` **Note:** by default, workflow runs are associated with the repository's default branch. If the repo's default branch is `develop` (as is currently the case for `MODFLOW-USGS/executables`, you will need to use the `--ref` (short `-r`) option to specify the `master` branch when triggering a release from the CLI. For instance: ```shell -gh workflow run continuous_integration.yml -R MODFLOW-USGS/executables -r master +gh workflow run release.yml -R MODFLOW-USGS/executables -r master ``` \ No newline at end of file diff --git a/scripts/build_executables.py b/scripts/build_executables.py new file mode 100755 index 0000000..b2f8a02 --- /dev/null +++ b/scripts/build_executables.py @@ -0,0 +1,98 @@ +import argparse +import sys +import subprocess +import textwrap +from pathlib import Path + +DEFAULT_RETRIES = 3 + + +def get_ostag() -> str: + """Determine operating system tag from sys.platform.""" + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform.startswith("win"): + return "win64" + elif sys.platform.startswith("darwin"): + return "mac" + raise ValueError(f"platform {sys.platform!r} not supported") + + +def get_cc() -> str: + """Determine operating system tag from sys.platform.""" + if sys.platform.startswith("linux"): + return "icc" + elif sys.platform.startswith("win"): + return "icl" + elif sys.platform.startswith("darwin"): + return "icc" + raise ValueError(f"platform {sys.platform!r} not supported") + + +def run_cmd(args) -> bool: + success = False + for idx in range(DEFAULT_RETRIES): + p = subprocess.run(args) + if p.returncode == 0: + success = True + break + print(f"{args[0]} run {idx + 1}/{DEFAULT_RETRIES} failed...rerunning") + return success + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog=f"Build MODFLOW-related binaries and metadata files", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( + """\ + Build MODFLOW-releated executables, shared libraries and metadata files with pymake. + """ + ), + ) + parser.add_argument( + "-p", + "--path", + required=False, + default=get_ostag(), + help="Path to create built binaries and metadata files", + ) + parser.add_argument( + "-r", + "--retries", + type=int, + required=False, + default=DEFAULT_RETRIES, + help="Number of times to retry a failed build", + + ) + args = parser.parse_args() + + # output path + path = Path(args.path) + path.mkdir(parents=True, exist_ok=True) + + # number of retries + retries = args.retries + + # C compiler + cc = get_cc() + + # create code.json + if not run_cmd([ + "make-code-json", + "-f", + str(path / "code.json"), + "--verbose", + ]): + raise RuntimeError(f"could not make code.json") + + # build binaries + if not run_cmd([ + "make-program", ":", + f"--appdir={path}", + "-fc=ifort", f"-cc={cc}", + f"--zip={path}.zip", + "--keep", + ]): + raise RuntimeError("could not build binaries") diff --git a/update_readme.py b/scripts/update_readme.py similarity index 52% rename from update_readme.py rename to scripts/update_readme.py index f80766a..177fe11 100644 --- a/update_readme.py +++ b/scripts/update_readme.py @@ -1,31 +1,17 @@ -import subprocess import pathlib as pl +proj_root = pl.Path(__file__).parent.parent target_file = pl.Path("code.md") TAG = "| Program | Version | UTC Date |" FILES = ("code.json", "code.md") -def _create_code_json() -> None: - subprocess.run( - [ - "make-code-json", - "-f", - f"code.json", - "--verbose", - ] - ) - - if not target_file.is_file(): - raise FileNotFoundError(f"{target_file} does not exist") - - def _update_readme() -> None: - with open("README.md", "r") as f: + with open(proj_root / "README.md", "r") as f: readme_md = f.read().splitlines() with open("code.md", "r") as f: code_md = f.read().splitlines() - with open("README.md", "w") as f: + with open(proj_root / "README.md", "w") as f: for line in readme_md: if TAG not in line: f.write(f"{line}\n") @@ -35,12 +21,5 @@ def _update_readme() -> None: break -def _clean_files() -> None: - for file in FILES: - pl.Path(file).unlink() - - if __name__ == "__main__": - _create_code_json() _update_readme() - _clean_files()