From b583567bd80e134231ea90cc064694d932b4312b Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 19 May 2019 12:46:39 +0100 Subject: [PATCH 01/14] automate binary releases for macOS & Windows [skip ci] --- .ci/azure-pipelines/matrix.yml | 3 +++ .ci/azure-pipelines/release.yml | 39 +++++++++++++++++++++++++++++++++ .ci/azure-pipelines/unix.yml | 5 +++++ .ci/azure-pipelines/windows.yml | 5 +++++ .ci/unix/delocate.sh | 19 ++++++++++++++++ azure-pipelines-release.yml | 17 ++++++++++++++ 6 files changed, 88 insertions(+) create mode 100644 .ci/azure-pipelines/release.yml create mode 100644 .ci/unix/delocate.sh create mode 100644 azure-pipelines-release.yml diff --git a/.ci/azure-pipelines/matrix.yml b/.ci/azure-pipelines/matrix.yml index 71a6ede9c1..b294b700ca 100644 --- a/.ci/azure-pipelines/matrix.yml +++ b/.ci/azure-pipelines/matrix.yml @@ -22,6 +22,8 @@ parameters: NESTING: basic GRIB1: 0 GRIB2: 1 + + RELEASE: false # Loop/matrix parameters. # Note that these may be overridden by the including yml (typically azure-pipelines.yml). @@ -40,6 +42,7 @@ jobs: - ${{ if not(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make'))) }}: - template: ${{ parameters.templates[OS_NAME] }} parameters: + RELEASE: ${{ parameters.RELEASE }} OS_NAME: ${{ OS_NAME }} BUILD_SYSTEM: ${{ BUILD_SYSTEM }} BUILD_TYPE: ${{ BUILD_TYPE }} diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml new file mode 100644 index 0000000000..e0215d9cce --- /dev/null +++ b/.ci/azure-pipelines/release.yml @@ -0,0 +1,39 @@ +steps: +- ${{ if eq(OS_NAME, 'Windows') }}: + - task: ArchiveFiles@2 + inputs: + rootFolderOrFile: build/install + includeRootFolder: false + archiveType: zip + archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(MODE)-$(NESTING)-$(BUILD_TYPE)-$(OS_NAME).zip' + displayName: Create distribution package + +- ${{ if not(eq(OS_NAME, 'Windows')) }}: + - bash: bash .ci/unix/delocate.sh + displayName: Delocate + + - task: ArchiveFiles@2 + inputs: + rootFolderOrFile: build/install + includeRootFolder: false + archiveType: tar + tarCompression: xz + archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(MODE)-$(NESTING)-$(BUILD_TYPE)-$(OS_NAME).tar.xz' + displayName: Create distribution package + +- bash: | + rename -f 'y/A-Z/a-z/' $(Build.ArtifactStagingDirectory)/* + displayName: Rename to lowercase + +- task: GithubRelease@0 + displayName: Create GitHub Draft Release + inputs: + githubConnection: WRF-CMake + repositoryName: WRF-CMake/WRF + # TODO we really want createOrEdit here like on Appveyor or Travis CI + #action: create + isDraft: true + isPreRelease: true + assetUploadMode: replace + assets: | + $(Build.ArtifactStagingDirectory)/* diff --git a/.ci/azure-pipelines/unix.yml b/.ci/azure-pipelines/unix.yml index e6ba555eb1..15646fcdb6 100644 --- a/.ci/azure-pipelines/unix.yml +++ b/.ci/azure-pipelines/unix.yml @@ -28,3 +28,8 @@ jobs: - ${{ if eq(parameters.WATS_RUN, 'true') }}: - template: wats_run.yml + + - ${{ if eq(parameters.RELEASE, 'true') }}: + - template: release.yml + parameters: + ${{ insert }}: ${{ parameters }} diff --git a/.ci/azure-pipelines/windows.yml b/.ci/azure-pipelines/windows.yml index a278cea81b..f2741bab34 100644 --- a/.ci/azure-pipelines/windows.yml +++ b/.ci/azure-pipelines/windows.yml @@ -43,3 +43,8 @@ jobs: - ${{ if eq(parameters.WATS_RUN, 'true') }}: - template: wats_run.yml + + - ${{ if eq(parameters.RELEASE, 'true') }}: + - template: release.yml + parameters: + ${{ insert }}: ${{ parameters }} \ No newline at end of file diff --git a/.ci/unix/delocate.sh b/.ci/unix/delocate.sh new file mode 100644 index 0000000000..8894e4f491 --- /dev/null +++ b/.ci/unix/delocate.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License. + +set -ex + +SCRIPTDIR=$(dirname "$0") +cd $SCRIPTDIR/../.. + +if [ "$(uname)" == "Darwin" ]; then + + brew upgrade python + pip3 install delocate + delocate-path build/install + +else + echo "Unknown OS: $(uname)" + exit 1 +fi \ No newline at end of file diff --git a/azure-pipelines-release.yml b/azure-pipelines-release.yml new file mode 100644 index 0000000000..fd648cb5f5 --- /dev/null +++ b/azure-pipelines-release.yml @@ -0,0 +1,17 @@ +# First, create a tag for the new version. +# Then, run this pipeline on the tag (will not deploy if untagged) +# to store binaries in a draft GitHub release. + +trigger: none +pr: none + +jobs: +- template: .ci/azure-pipelines/matrix.yml + parameters: + OS_NAMES: [Windows, macOS] # TODO Linux + BUILD_SYSTEMS: [CMake] + BUILD_TYPES: [Debug, Release] + MODES: [serial, dmpar] + WATS_DIFF: false + WATS_PLOTS: false + RELEASE: true From a959702caac0ddcfcbaeeb977426c79c6b5a653b Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 19 May 2019 12:52:27 +0100 Subject: [PATCH 02/14] fix [skip ci] --- .ci/azure-pipelines/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index e0215d9cce..e2438c6597 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -1,5 +1,5 @@ steps: -- ${{ if eq(OS_NAME, 'Windows') }}: +- ${{ if eq(parameters.OS_NAME, 'Windows') }}: - task: ArchiveFiles@2 inputs: rootFolderOrFile: build/install @@ -8,7 +8,7 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(MODE)-$(NESTING)-$(BUILD_TYPE)-$(OS_NAME).zip' displayName: Create distribution package -- ${{ if not(eq(OS_NAME, 'Windows')) }}: +- ${{ if not(eq(parameters.OS_NAME, 'Windows')) }}: - bash: bash .ci/unix/delocate.sh displayName: Delocate From afc1310a3f6e2f9e4eafaec2e050c95eec964d98 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 19 May 2019 13:19:22 +0100 Subject: [PATCH 03/14] fix [skip ci] --- .ci/azure-pipelines/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index e2438c6597..56dc7f12d8 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -28,7 +28,7 @@ steps: - task: GithubRelease@0 displayName: Create GitHub Draft Release inputs: - githubConnection: WRF-CMake + githubConnection: WRF-CMake-releases repositoryName: WRF-CMake/WRF # TODO we really want createOrEdit here like on Appveyor or Travis CI #action: create From ba0af84e54920e23c67fabbbb1818b30f743bc0d Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 19 May 2019 13:31:58 +0100 Subject: [PATCH 04/14] add comment [skip ci] --- azure-pipelines-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines-release.yml b/azure-pipelines-release.yml index fd648cb5f5..915db3b208 100644 --- a/azure-pipelines-release.yml +++ b/azure-pipelines-release.yml @@ -2,6 +2,8 @@ # Then, run this pipeline on the tag (will not deploy if untagged) # to store binaries in a draft GitHub release. +# Note that these values are also overridden to "none" in Azure DevOps's UI to +# prevent others to run this pipeline by changing the triggers below and opening a PR. trigger: none pr: none From 3872f502fabc1f0e94e64ab8bd085576c087b29e Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 19 May 2019 15:10:37 +0100 Subject: [PATCH 05/14] fixes [skip ci] --- .ci/azure-pipelines/release.yml | 4 ++-- .ci/unix/delocate.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index 56dc7f12d8..b603cb40a3 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -21,8 +21,8 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(MODE)-$(NESTING)-$(BUILD_TYPE)-$(OS_NAME).tar.xz' displayName: Create distribution package -- bash: | - rename -f 'y/A-Z/a-z/' $(Build.ArtifactStagingDirectory)/* +- pwsh: | + dir $(Build.ArtifactStagingDirectory) -r | % { if ($_.Name -cne $_.Name.ToLower()) { ren $_.FullName $_.Name.ToLower() } } displayName: Rename to lowercase - task: GithubRelease@0 diff --git a/.ci/unix/delocate.sh b/.ci/unix/delocate.sh index 8894e4f491..35015cf3ef 100644 --- a/.ci/unix/delocate.sh +++ b/.ci/unix/delocate.sh @@ -9,7 +9,6 @@ cd $SCRIPTDIR/../.. if [ "$(uname)" == "Darwin" ]; then - brew upgrade python pip3 install delocate delocate-path build/install From 06ce263f0f5939c5ba5483d832f701e62f84917c Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 19 May 2019 15:43:57 +0100 Subject: [PATCH 06/14] fix [skip ci] --- .ci/azure-pipelines/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index b603cb40a3..c766c5d1e3 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -30,8 +30,8 @@ steps: inputs: githubConnection: WRF-CMake-releases repositoryName: WRF-CMake/WRF - # TODO we really want createOrEdit here like on Appveyor or Travis CI - #action: create + action: edit # will create if not existing + tag: $(Build.SourceBranchName) isDraft: true isPreRelease: true assetUploadMode: replace From d0a49b447a52b0ed186325972650aa5115ca5f36 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 20 May 2019 10:19:17 +0100 Subject: [PATCH 07/14] fix [skip ci] --- .ci/azure-pipelines/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index c766c5d1e3..631ee74aaa 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -28,7 +28,7 @@ steps: - task: GithubRelease@0 displayName: Create GitHub Draft Release inputs: - githubConnection: WRF-CMake-releases + gitHubConnection: WRF-CMake-releases repositoryName: WRF-CMake/WRF action: edit # will create if not existing tag: $(Build.SourceBranchName) From 81e10677b3482aaebaceec89a260f5063860ca5f Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 20 May 2019 19:33:00 +0100 Subject: [PATCH 08/14] remove changelog [skip ci] --- .ci/azure-pipelines/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index 631ee74aaa..e579d57bf1 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -34,6 +34,7 @@ steps: tag: $(Build.SourceBranchName) isDraft: true isPreRelease: true + addChangeLog: false assetUploadMode: replace assets: | $(Build.ArtifactStagingDirectory)/* From 38774ff960fe8ef513cb0726b6985bb56a487611 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 20 May 2019 19:37:16 +0100 Subject: [PATCH 09/14] store artifact --- .ci/azure-pipelines/release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index e579d57bf1..291b2aec98 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -25,6 +25,13 @@ steps: dir $(Build.ArtifactStagingDirectory) -r | % { if ($_.Name -cne $_.Name.ToLower()) { ren $_.FullName $_.Name.ToLower() } } displayName: Rename to lowercase +# not strictly needed +- task: PublishBuildArtifacts@1 + displayName: Store distribution package + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: dist_$(OS_NAME)_$(BUILD_SYSTEM)_$(BUILD_TYPE)_$(MODE) + - task: GithubRelease@0 displayName: Create GitHub Draft Release inputs: From 59fa1589363f91b7964f87f692ca874c5f1cf1f7 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 20 May 2019 20:33:58 +0100 Subject: [PATCH 10/14] fix path [skip ci] --- .ci/unix/delocate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/unix/delocate.sh b/.ci/unix/delocate.sh index 35015cf3ef..5a1376f144 100644 --- a/.ci/unix/delocate.sh +++ b/.ci/unix/delocate.sh @@ -10,7 +10,7 @@ cd $SCRIPTDIR/../.. if [ "$(uname)" == "Darwin" ]; then pip3 install delocate - delocate-path build/install + delocate-path build/install/main else echo "Unknown OS: $(uname)" From 4f90e267d4f7dec02fe3be0c089a0052eb1e7077 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Wed, 22 May 2019 22:18:04 +0100 Subject: [PATCH 11/14] text [skip ci] --- azure-pipelines-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-release.yml b/azure-pipelines-release.yml index 915db3b208..955ec1951f 100644 --- a/azure-pipelines-release.yml +++ b/azure-pipelines-release.yml @@ -1,9 +1,9 @@ # First, create a tag for the new version. -# Then, run this pipeline on the tag (will not deploy if untagged) -# to store binaries in a draft GitHub release. +# Then, run this pipeline on the tag to store binaries in a draft GitHub release. # Note that these values are also overridden to "none" in Azure DevOps's UI to # prevent others to run this pipeline by changing the triggers below and opening a PR. +# It still wouldn't cause harm as secrets (GitHub credentials) are not available to PRs from forks. trigger: none pr: none From fdc36943974eb4f166bea214d576fc43beec2c18 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Fri, 24 May 2019 22:48:15 +0100 Subject: [PATCH 12/14] change release filename format --- .ci/azure-pipelines/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index 291b2aec98..a51ae6fee0 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -5,7 +5,7 @@ steps: rootFolderOrFile: build/install includeRootFolder: false archiveType: zip - archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(MODE)-$(NESTING)-$(BUILD_TYPE)-$(OS_NAME).zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).zip' displayName: Create distribution package - ${{ if not(eq(parameters.OS_NAME, 'Windows')) }}: @@ -18,7 +18,7 @@ steps: includeRootFolder: false archiveType: tar tarCompression: xz - archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(MODE)-$(NESTING)-$(BUILD_TYPE)-$(OS_NAME).tar.xz' + archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).tar.xz' displayName: Create distribution package - pwsh: | From f89fdeff6f9ed07cf31d36d3fdaa059270dedada Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Sun, 30 Jun 2019 20:57:22 +0100 Subject: [PATCH 13/14] debug output --- .ci/unix/delocate.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/unix/delocate.sh b/.ci/unix/delocate.sh index 5a1376f144..181d3e34ec 100644 --- a/.ci/unix/delocate.sh +++ b/.ci/unix/delocate.sh @@ -10,7 +10,9 @@ cd $SCRIPTDIR/../.. if [ "$(uname)" == "Darwin" ]; then pip3 install delocate + delocate-listdeps --all --depending build/install/main delocate-path build/install/main + delocate-listdeps --all --depending build/install/main else echo "Unknown OS: $(uname)" From be4816a32c0e5232e75d8eefc3e3a0f0a5af3b33 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Wed, 3 Jul 2019 22:44:41 +0100 Subject: [PATCH 14/14] possible fix for delocate using gcc@8 for wrf together with netcdf-fortran (which links against gcc (=9)) meant we have two libgfortran's which delocate can't deal with. --- .ci/azure-pipelines/matrix.yml | 4 ++-- .ci/unix/Brewfile | 2 +- .travis.yml | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/azure-pipelines/matrix.yml b/.ci/azure-pipelines/matrix.yml index f07ecd98e1..8af851e0f5 100644 --- a/.ci/azure-pipelines/matrix.yml +++ b/.ci/azure-pipelines/matrix.yml @@ -11,8 +11,8 @@ parameters: CC: gcc-8 FC: gfortran-8 macOS: - CC: gcc-8 - FC: gfortran-8 + CC: gcc-9 + FC: gfortran-9 Windows: CC: gcc FC: gfortran diff --git a/.ci/unix/Brewfile b/.ci/unix/Brewfile index 1b3ca7bb02..d5c21ded8c 100644 --- a/.ci/unix/Brewfile +++ b/.ci/unix/Brewfile @@ -1,5 +1,5 @@ brew "coreutils" -brew "gcc@8" +brew "gcc" brew "netcdf" brew "jasper" brew "libpng" diff --git a/.travis.yml b/.travis.yml index dde37c3e5b..e840675011 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,8 @@ jobs: env: - BUILD_SYSTEM=Make - BUILD_TYPE=Debug - - CC=gcc-8 # GNU via Homebrew - - FC=gfortran-8 + - CC=gcc-9 # GNU via Homebrew + - FC=gfortran-9 - MODE=serial - stage: build @@ -48,8 +48,8 @@ jobs: env: - BUILD_SYSTEM=Make - BUILD_TYPE=Debug - - CC=gcc-8 # GNU via Homebrew - - FC=gfortran-8 + - CC=gcc-9 # GNU via Homebrew + - FC=gfortran-9 - MODE=dmpar # CMake builds @@ -89,8 +89,8 @@ jobs: env: - BUILD_SYSTEM=CMake - BUILD_TYPE=Debug - - CC=gcc-8 # GNU via Homebrew - - FC=gfortran-8 + - CC=gcc-9 # GNU via Homebrew + - FC=gfortran-9 - GRIB1=1 - GRIB2=1 - MODE=serial @@ -104,7 +104,7 @@ jobs: - BUILD_SYSTEM=CMake - BUILD_TYPE=Debug - CC=clang # Apple Clang - - FC=gfortran-8 + - FC=gfortran-9 - GRIB1=1 - GRIB2=1 - MODE=dmpar