ci: add CI to build for all OSs with Intel (#8)

.github/common/build_executables.py

@@ -0,0 +1,50 @@
+import os
+import sys
+import pymake
+import subprocess
+import pathlib as pl
+
+
+def get_ostag() -> str:
+ """Determine operating system tag from sys.platform."""
+ if sys.platform.startswith("linux"):
+ return "linux"
+ elif sys.platform.startswith("win"):
+ return "win64"
+ elif sys.platform.startswith("darwin"):
+ return "mac"
+ raise ValueError(f"platform {sys.platform!r} not supported")
+
+
+def get_cctag() -> str:
+ """Determine operating system tag from sys.platform."""
+ if sys.platform.startswith("linux"):
+ return "icc"
+ elif sys.platform.startswith("win"):
+ return "icl"
+ elif sys.platform.startswith("darwin"):
+ return "icc"
+ raise ValueError(f"platform {sys.platform!r} not supported")
+
+
+if __name__ == "__main__":
+ path = (
+ pl.Path(os.path.dirname(pymake.__file__)) / "../"
+ ).resolve() / "examples/buildall.py"
+
+ cmds = [
+ "python",
+ path,
+ f"--appdir={get_ostag()}",
+ "-fc=ifort",
+ f"-cc={get_cctag()}",
+ f"--zip={get_ostag()}.zip",
+ "--keep",
+ ]
+
+ retries = 3
+ for idx in range(retries):
+ p = subprocess.run(cmds)
+ if p.returncode == 0:
+ break
+ print(f"run {idx + 1}/{retries} failed...rerunning")

.github/common/install-python.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+python -m pip install --upgrade pip
+pip install wheel
+pip install https://github.com/modflowpy/pymake/zipball/develop
+

.github/intel-scripts/README.md

@@ -0,0 +1,17 @@
+These scripts install the ifort compiler on the CI machines.
+
+The content has been directly copied from the Intel OneAPI-CI examples, found
+here:
+
+https://github.com/oneapi-src/oneapi-ci
+
+Note that on Linux and macOS, shell scripts must be marked as executable: not
+doing so for these scripts will result in a permission error during the GitHub
+Action. Windows does not have such a permission system, but you can mark a 
+script nonetheless with git:
+
+```sh
+git update-index --chmod=+x {name_of_script}
+```
+
+Then commit and push.

.github/intel-scripts/cache_exclude_linux.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+#shellcheck disable=SC2010
+LATEST_VERSION=$(ls -1 /opt/intel/oneapi/compiler/ | grep -v latest | sort | tail -1)
+
+sudo rm -rf /opt/intel/oneapi/compiler/"$LATEST_VERSION"/linux/compiler/lib/ia32_lin
+sudo rm -rf /opt/intel/oneapi/compiler/"$LATEST_VERSION"/linux/bin/ia32
+sudo rm -rf /opt/intel/oneapi/compiler/"$LATEST_VERSION"/linux/lib/emu
+sudo rm -rf /opt/intel/oneapi/compiler/"$LATEST_VERSION"/linux/lib/oclfpga

.github/intel-scripts/cache_exclude_windows.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+#shellcheck disable=SC2010
+LATEST_VERSION=$(ls -1 "C:\Program Files (x86)\Intel\oneAPI\compiler" | grep -v latest | sort | tail -1)
+
+rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\'$LATEST_VERSION'\windows\compiler\lib\ia32_win"
+rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\'$LATEST_VERSION'\windows\bin\intel64_ia32"
+rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\'$LATEST_VERSION'\windows\lib\emu"
+rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\'$LATEST_VERSION'\windows\lib\oclfpga"
+rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\'$LATEST_VERSION'\windows\lib\ocloc"
+rm -rf "C:\Program Files (x86)\Intel\oneAPI\compiler\'$LATEST_VERSION'\windows\lib\x86"

.github/intel-scripts/ifortvars_linux.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+#shellcheck disable=SC2010
+LATEST_VERSION=$(ls -1 /opt/intel/oneapi/compiler/ | grep -v latest | sort | tail -1)
+# shellcheck source=/dev/null
+source /opt/intel/oneapi/compiler/"$LATEST_VERSION"/env/vars.sh
+
+# print intel compiler versions
+ifort --version
+icc --version
+
+# build executables
+python .github/common/build_executables.py
+

.github/intel-scripts/ifortvars_macos.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+source /opt/intel/oneapi/setvars.sh
+
+# print intel compiler versions
+ifort --version
+icc --version
+
+# build executables
+python .github/common/build_executables.py
+

.github/intel-scripts/ifortvars_windows.bat

@@ -0,0 +1,16 @@
+REM SPDX-FileCopyrightText: 2020 Intel Corporation
+REM
+REM SPDX-License-Identifier: MIT
+
+set LANGUAGE=%1
+set VS_VER=%2
+
+IF "%VS_VER%"=="2017_build_tools" (
+@call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+)
+
+IF "%VS_VER%"=="2019_build_tools" (
+@call "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+)
+for /f "tokens=* usebackq" %%f in (`dir /b "C:\Program Files (x86)\Intel\oneAPI\compiler\" ^| findstr /V latest ^| sort`) do @set "LATEST_VERSION=%%f"
+@call "C:\Program Files (x86)\Intel\oneAPI\compiler\%LATEST_VERSION%\env\vars.bat"

.github/intel-scripts/install_linux.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+URL=$1
+COMPONENTS=$2
+
+curl --output webimage.sh --url "$URL" --retry 5 --retry-delay 5
+chmod +x webimage.sh
+./webimage.sh -x -f webimage_extracted --log extract.log
+rm -rf webimage.sh
+WEBIMAGE_NAME=$(ls -1 webimage_extracted/)
+if [ -z "$COMPONENTS" ]; then
+ sudo webimage_extracted/"$WEBIMAGE_NAME"/bootstrapper -s --action install --eula=accept --continue-with-optional-error=yes --log-dir=.
+ installer_exit_code=$?
+else
+ sudo webimage_extracted/"$WEBIMAGE_NAME"/bootstrapper -s --action install --components="$COMPONENTS" --eula=accept --continue-with-optional-error=yes --log-dir=.
+ installer_exit_code=$?
+fi
+rm -rf webimage_extracted
+exit $installer_exit_code

.github/intel-scripts/install_macos.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+URL=$1
+COMPONENTS=$2
+
+curl --output webimage.dmg --url "$URL" --retry 5 --retry-delay 5
+hdiutil attach webimage.dmg
+if [ -z "$COMPONENTS" ]; then
+ sudo /Volumes/"$(basename "$URL" .dmg)"/bootstrapper.app/Contents/MacOS/bootstrapper -s --action install --eula=accept --continue-with-optional-error=yes --log-dir=.
+ installer_exit_code=$?
+else
+ sudo /Volumes/"$(basename "$URL" .dmg)"/bootstrapper.app/Contents/MacOS/bootstrapper -s --action install --components="$COMPONENTS" --eula=accept --continue-with-optional-error=yes --log-dir=.
+ installer_exit_code=$?
+fi
+hdiutil detach /Volumes/"$(basename "$URL" .dmg)" -quiet
+exit $installer_exit_code

.github/intel-scripts/install_windows.bat

@@ -0,0 +1,15 @@
+REM SPDX-FileCopyrightText: 2020 Intel Corporation
+REM
+REM SPDX-License-Identifier: MIT
+
+set URL=%1
+set COMPONENTS=%2
+
+curl.exe --output webimage.exe --url %URL% --retry 5 --retry-delay 5
+start /b /wait webimage.exe -s -x -f webimage_extracted --log extract.log
+del webimage.exe
+if "%COMPONENTS%"=="" (
+ webimage_extracted\bootstrapper.exe -s --action install --eula=accept --continue-with-optional-error=yes -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 --log-dir=.
+) else (
+ webimage_extracted\bootstrapper.exe -s --action install --components=%COMPONENTS% --eula=accept --continue-with-optional-error=yes -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 --log-dir=.
+)

.github/workflows/continuous_integration.yml

@@ -0,0 +1,122 @@
+name: executables continuous integration
+
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+jobs:
+ executables-intel:
+ name: pymake CI intel on different OSs
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - os: ubuntu-latest
+ os_script: ifortvars_linux.sh
+ - os: macos-latest
+ os_script: ifortvars_macos.sh
+ - os: windows-2019
+ os_script: ifortvars_windows.bat
+ defaults:
+ run:
+ shell: bash -l {0}
+ env:
+ WINDOWS_HPCKIT_URL: https://registrationcenter-download.intel.com/akdlm/irc_nas/17392/w_HPCKit_p_2021.1.0.2682_offline.exe
+ LINUX_HPCKIT_URL: https://registrationcenter-download.intel.com/akdlm/irc_nas/17427/l_HPCKit_p_2021.1.0.2684_offline.sh
+ MACOS_HPCKIT_URL: https://registrationcenter-download.intel.com/akdlm/irc_nas/17398/m_HPCKit_p_2021.1.0.2681_offline.dmg
+ WINDOWS_FORTRAN_COMPONENTS: intel.oneapi.win.ifort-compiler
+ LINUX_FORTRAN_COMPONENTS_WEB: intel.oneapi.lin.ifort-compiler
+ MACOS_FORTRAN_COMPONENTS: intel.oneapi.mac.ifort-compiler
+ WINDOWS_CPP_COMPONENTS: intel.oneapi.win.cpp-compiler
+ LINUX_CPP_COMPONENTS_WEB: intel.oneapi.lin.dpcpp-cpp-compiler-pro
+ MACOS_CPP_COMPONENTS: intel.oneapi.mac.cpp-compiler
+ FC: ifort
+
+ steps:
+ - name: Checkout repo
+ uses: actions/[email protected]
+
+ - name: Install Conda environment
+ uses: mamba-org/provision-with-micromamba@main
+ with:
+ cache-downloads: true
+ cache-env: true
+
+ - name: Print Python package versions
+ run: |
+ pip list
+
+ - name: Install python packages
+ run: |
+ .github/common/install-python.sh
+
+ - name: cache install intel compilers on linux
+ if: runner.os == 'Linux'
+ id: cache-install-linux
+ uses: actions/cache@v2
+ with:
+ path: |
+ /opt/intel/oneapi/compiler
+ key: install-${{ env.LINUX_HPCKIT_URL }}-${{ env.LINUX_CPP_COMPONENTS_WEB }}-${{ env.LINUX_FORTRAN_COMPONENTS_WEB }}-compiler-${{ hashFiles('**/.github/intel-scripts/cache_exclude_linux.sh') }}
+
+ - name: install intel compilers on linux
+ if: runner.os == 'Linux' && steps.cache-install-linux.outputs.cache-hit != 'true'
+ run: |
+ .github/intel-scripts/install_linux.sh $LINUX_HPCKIT_URL $LINUX_CPP_COMPONENTS_WEB:$LINUX_FORTRAN_COMPONENTS_WEB
+
+ - name: cache install intel compilers on macos
+ if: runner.os == 'macOS'
+ id: cache-install-macos
+ uses: actions/cache@v2
+ with:
+ path: /opt/intel/oneapi
+ key: install-${{ env.MACOS_HPCKIT_URL }}--${{ env.MACOS_CPP_COMPONENTS }}-${{ env.MACOS_FORTRAN_COMPONENTS }}
+
+ - name: install intel compilers on macos
+ if: runner.os == 'macOS' && steps.cache-install-macos.outputs.cache-hit != 'true'
+ run: |
+ .github/intel-scripts/install_macos.sh $MACOS_HPCKIT_URL $MACOS_CPP_COMPONENTS:$MACOS_FORTRAN_COMPONENTS
+
+ - name: cache install ifort on windows
+ if: runner.os == 'Windows'
+ id: cache-install-windows
+ uses: actions/cache@v2
+ with:
+ path: C:\Program Files (x86)\Intel\oneAPI\compiler
+ key: install-${{ env.WINDOWS_HPCKIT_URL }}-${{ env.WINDOWS_CPP_COMPONENTS }}-${{ env.WINDOWS_FORTRAN_COMPONENTS }}-compiler-${{ hashFiles('**/.github/intel-scripts/cache_exclude_windows.sh') }}
+
+ - name: install intel compilers on windows
+ if: runner.os == 'Windows' && steps.cache-install-windows.outputs.cache-hit != 'true'
+ run: |
+ .github/intel-scripts/install_windows.bat $WINDOWS_HPCKIT_URL $WINDOWS_CPP_COMPONENTS:$WINDOWS_FORTRAN_COMPONENTS
+
+ - name: activate intel and test on linux and macOS
+ if: runner.os == 'Linux' || runner.os == 'macOS'
+ env:
+ CC: icc
+ run: |
+ .github/intel-scripts/${{ matrix.os_script }}
+
+ - name: activate intel and test on Windows
+ if: runner.os == 'Windows'
+ env:
+ CC: icl
+ shell: cmd
+ run: |
+ call ".github/intel-scripts/ifortvars_windows.bat"
+ python .github/common/build_executables.py
+
+ - name: list whats available - temporary
+ run: |
+ ls -a 
+
+ - name: exclude unused files from cache on windows
+ if: runner.os == 'Windows' && steps.cache-install-windows.outputs.cache-hit != 'true'
+ run: |
+ .github/intel-scripts/cache_exclude_windows.sh
+
+ - name: exclude unused files from cache on linux
+ if: runner.os == 'Linux' && steps.cache-install-linux.outputs.cache-hit != 'true'
+ run: |
+ .github/intel-scripts/cache_exclude_linux.sh