From e84ff4a20c14693892808e682ca04533ad342419 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 15 Feb 2024 15:00:17 -0500 Subject: [PATCH] refactor: restore double precision executables (#24) * make dp versions of mf2005, mfusg, mflgr, and mfnwt * uses new --double argument for pymake make-program * add modflow-devtools to requirements.txt, use ostags module * use develop branches of pymake and devtools in requirements.txt * rename scripts/build_executables.py -> scripts/build_programs.py * test mf6 built by pymake not by meson (needed for mf5to6) in integration.yml * check that test distribution zipfile exists in integration.yml * update .gitignore * format with black --- .github/workflows/integration.yml | 73 +++++++++------ .github/workflows/release.yml | 2 +- .gitignore | 5 +- requirements.txt | 3 +- ...build_executables.py => build_programs.py} | 93 +++++++++++-------- 5 files changed, 106 insertions(+), 70 deletions(-) rename scripts/{build_executables.py => build_programs.py} (52%) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 589efab..6da3ea5 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -48,41 +48,62 @@ jobs: with: compiler: intel-classic version: 2021.7 - + + # only necessary because we need mf5to6 for mf6 autotests - name: Build modflow6 working-directory: modflow6 run: | meson setup builddir --prefix=$(pwd) --libdir=bin -Ddebug=false meson install -C builddir - - - name: Show meson build log - if: failure() - working-directory: modflow6 - run: cat builddir/meson-logs/meson-log.txt - - name: Unit test modflow6 - working-directory: modflow6 - run: meson test --verbose --no-rebuild -C builddir + - name: Build programs + run: | + # build programs + mkdir programs + python executables/scripts/build_programs.py -p programs - - name: Update FloPy - working-directory: modflow6/autotest - run: python update_flopy.py + # move programs where mf6 autotests expect them + mkdir modflow6/bin/downloaded + cp programs/* modflow6/bin/downloaded - - name: Get executables - working-directory: modflow6/autotest - env: - GITHUB_TOKEN: ${{ github.token }} - run: pytest -v -s get_exes.py + # move mf6 binaries to top-level bindir in mf6 repo + if [[ "$RUNNER_OS" == "Windows" ]]; then + eext=".exe" + oext=".dll" + elif [[ "$RUNNER_OS" == "Linux" ]]; then + eext="" + oext=".so" + else + eext="" + oext=".dylib" + fi + cp "programs/mf6$eext" modflow6/bin + cp "programs/libmf6$oext" modflow6/bin + cp "programs/zbud6$eext" modflow6/bin + + # set execute permissions + if [[ "$RUNNER_OS" != "Windows" ]]; then + sudo chmod +x modflow6/bin/* + sudo chmod +x modflow6/bin/downloaded/* + fi + + - name: Upload programs + uses: actions/upload-artifact@v3 + with: + name: programs + path: programs.zip - - name: Build executables - working-directory: executables/scripts - run: python build_executables.py -p ../../modflow6/bin/downloaded - - - name: Set executable permission - if: runner.os != 'Windows' - working-directory: modflow6/bin/downloaded - run: sudo chmod +x * + - name: Upload metadata + if: runner.os == 'Linux' + uses: actions/upload-artifact@v3 + with: + name: metadata + path: | + ./code.json + ./code.md - name: Test modflow6 working-directory: modflow6/autotest - run: pytest -v -n auto --durations 0 + run: | + python update_flopy.py + pytest -v -n auto -k "not gwe" -m "not developmode" --durations 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e64967d..3003251 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,7 +54,7 @@ jobs: pip list - name: Build executables - run: python scripts/build_executables.py + run: python scripts/build_programs.py - name: Upload distribution archive uses: actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index d953ad5..db86fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,9 @@ target/ # PyCharm .idea/ -# files in the examples directory +# Local testing folders temp/ +bin/ +# Python virtual environments +venv/ diff --git a/requirements.txt b/requirements.txt index 069b1f5..c284a46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ wheel -https://github.com/modflowpy/pymake/zipball/master \ No newline at end of file +https://github.com/modflowpy/pymake/zipball/develop +https://github.com/MODFLOW-USGS/modflow-devtools/zipball/develop \ No newline at end of file diff --git a/scripts/build_executables.py b/scripts/build_programs.py similarity index 52% rename from scripts/build_executables.py rename to scripts/build_programs.py index 96ebb93..f13fca9 100755 --- a/scripts/build_executables.py +++ b/scripts/build_programs.py @@ -4,22 +4,19 @@ import textwrap from pathlib import Path +from modflow_devtools.ostags import get_modflow_ostag + DEFAULT_RETRIES = 3 +DBL_PREC_PROGRAMS = ["mf2005", "mflgr", "mfnwt", "mfusg"] -def get_ostag() -> str: - """Determine operating system tag from sys.platform.""" - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform.startswith("win"): - return "win64" - elif sys.platform.startswith("darwin"): - return "mac" - raise ValueError(f"platform {sys.platform!r} not supported") +def get_fc() -> str: + """Determine Intel Fortran compiler to use based on the current platform.""" + return "ifort" def get_cc() -> str: - """Determine operating system tag from sys.platform.""" + """Determine Intel C compiler to use based on the current platform.""" if sys.platform.startswith("linux"): return "icc" elif sys.platform.startswith("win"): @@ -46,7 +43,7 @@ def run_cmd(args) -> bool: formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent( """\ - Build MODFLOW-releated executables, shared libraries and metadata files with pymake. + Build MODFLOW-related executables, shared libraries and metadata files with pymake. """ ), ) @@ -60,7 +57,7 @@ def run_cmd(args) -> bool: "-p", "--path", required=False, - default=get_ostag(), + default=get_modflow_ostag(), help="Path to create built binaries and metadata files", ) parser.add_argument( @@ -70,40 +67,54 @@ def run_cmd(args) -> bool: required=False, default=DEFAULT_RETRIES, help="Number of times to retry a failed build", - ) args = parser.parse_args() - - # whether to recreate existing binaries keep = bool(args.keep) - - # output path path = Path(args.path) path.mkdir(parents=True, exist_ok=True) - - # number of retries + zip_path = Path(path).with_suffix(".zip") + dp_programs = ",".join(DBL_PREC_PROGRAMS) retries = args.retries - - # C compiler + fc = get_fc() cc = get_cc() - # create code.json - if not run_cmd([ - "make-code-json", - "-f", - str(path / "code.json"), - "--verbose", - ]): - raise RuntimeError(f"could not make code.json") - - # build binaries - build_args = [ - "make-program", ":", - f"--appdir={path}", - "-fc=ifort", f"-cc={cc}", - f"--zip={path}.zip", - ] - if keep: - build_args.append("--keep") - if not run_cmd(build_args): - raise RuntimeError("could not build binaries") + assert run_cmd( + [ + "make-code-json", + "-f", + str(path / "code.json"), + "--verbose", + ] + ), "could not make code.json" + assert run_cmd( + [ + "make-program", + ":", + "--appdir", + path, + "-fc", + fc, + "-cc", + cc, + "--zip", + f"{path}.zip", + ] + + (["--keep"] if keep else []) + ), "could not build default precision binaries" + assert run_cmd( + [ + "make-program", + dp_programs, + "--appdir", + path, + "--double", + "--keep", + "-fc", + fc, + "-cc", + cc, + "--zip", + str(zip_path), + ] + ), f"could not build double precision binaries: {dp_programs}" + assert zip_path.is_file(), "could not build distribution zipfile"