Fix: GH workflows (#1033)

* Fix: GH workflows

---------

Co-authored-by: Michael Kaiser <[email protected]>

.github/actions/setup-language/action.yml

@@ -0,0 +1,31 @@
+name: 'Setup Language'
+description: 'Set up the environment for the specified language'
+inputs:
+ language:
+ description: 'The programming language to set up'
+ required: true
+runs:
+ using: 'composite'
+ steps:
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ if: ${{ matrix.language == 'go' }}
+ with:
+ go-version: '1.20'
+ - uses: actions/setup-node@v4
+ if: ${{matrix.language == 'typescript'}}
+ with:
+ node-version: 18
+ - uses: actions/setup-java@v4
+ if: ${{matrix.language == 'java'}}
+ with:
+ distribution: 'corretto'
+ java-version: '17'
+ - uses: actions/setup-python@v5
+ if: ${{matrix.language == 'python'}}
+ with:
+ python-version: '3.10'
+ - uses: actions/setup-dotnet@v4
+ if: ${{matrix.language == 'csharp'}}
+ with:
+ dotnet-version: 8

.github/workflows/build-pull-request.yml

@@ -1,92 +1,107 @@
+# Name of the workflow
 name: Build Pull Request
 
-on: [pull_request]
+# Trigger the workflow on pull requests
+on:
+ pull_request:
+ # Only run the workflow when non-Markdown files are changed
+ paths:
+ - '**'
 
 jobs:
+
  build:
+ # Run the job on the latest Ubuntu runner
  runs-on: ubuntu-latest
+
+ # Define a matrix strategy to run the job for multiple languages
  strategy:
  matrix:
- language: ['csharp', 'go','python', 'java', 'typescript']
+ language: ['csharp', 'go', 'python', 'java', 'typescript']
+
  steps:
- - name: Set up Go
- uses: actions/setup-go@v5
- if: ${{ matrix.language == 'go' }}
- with:
- go-version: '1.20'
- - uses: actions/setup-node@v4
- if: ${{matrix.language == 'typescript'}}
- with:
- node-version: 18
- - uses: actions/setup-java@v4
- if: ${{matrix.language == 'java'}}
- with:
- distribution: 'corretto'
- java-version: '17'
- - uses: actions/setup-python@v5
- if: ${{matrix.language == 'python'}}
- with:
- python-version: '3.10'
- - uses: actions/setup-dotnet@v4
- if: ${{matrix.language == 'csharp'}}
- with:
- dotnet-version: 8
+ # Checkout the code from the repository
  - name: Checkout code
  uses: actions/checkout@v4
+
+ # Set up the required environment for the specified language
+ - name: Setup Language
+ uses: ./.github/actions/setup-language
  with:
- fetch-depth: 0
+ language: ${{ matrix.language }}
+
+ # Get the list of changed files, excluding Markdown files
  - name: Get changed files
- id: changed-files-specific
- uses: tj-actions/changed-files@v44
+ id: changed-files
+ uses: tj-actions/changed-files@v35
  with:
- files: ${{matrix.language}}/**
- files_ignore: "*.md"
- - name: list changed files
- if: steps.changed-files-specific.outputs.any_changed == 'true'
- run: |
- for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
- echo "$file was changed"
- done
+ files: ${{ matrix.language }}/**
+ files_ignore: '**/*.md'
+
+ # Build the changed files for the specified language
  - name: Build changed files
- if: steps.changed-files-specific.outputs.any_changed == 'true'
+ if: steps.changed-files.outputs.any_changed == 'true'
  run: |
- echo "::group::Build changes for ${{ matrix.language }}"
- echo ""
- buildlang="${{ matrix.language }}"
-
- echo "# Build summary for ${buildlang}" >> $GITHUB_STEP_SUMMARY
- buildpath=""
- # For each file that is in the changed set
- for file in ${{ steps.changed-files-specific.outputs.all_modified_files }}
- do
- echo "processing ${file}"
- # Split the path into the language, example, and 
- IFS="/" read path1 path2 extra <<<"$file"
- # $path1 is the language name e.g. 'typescript'
- # $path2 is the fist directory under the language
- # $extra is everything else after path2
- if [[ $path1 != $buildlang ]]; then
- continue
- fi 
- if [[ "$buildpath" == "$path1/$path2" ]]; then
- continue
+ # Function to build a single file
+ build_file() {
+ echo "Build File $1"
+ local file="$1"
+ IFS="/" read -ra path_parts <<< "$file"
+ language=${path_parts[0]}
+
+ # Skip files that don't belong to the current language
+ if [[ $language != ${{ matrix.language }} ]]; then
+ return 0
  fi
- buildpath=$path1/$path2
- echo "Build Path ${buildpath}"
- 
- # make sure there's nothing funny left over.
- git clean -dfx
-
- # we should pass both $path2 and $extra to the build script
- # for example:
- # scripts/build-typescript.sh ecs example-name/index.ts for typescripts/ecs/example/index.ts 
- # scripts/build-typescript.sh example index.ts for typescripts/example/index.ts 
- scripts/build-${buildlang}.sh $path2 $extra
- 
- if [[ $? == 0 ]]; then
- echo "- :o: $buildpath" >> $GITHUB_STEP_SUMMARY
+
+ echo "Build Path $file"
+
+ # Run the build script for the current language, passing the project directory and extra path
+ echo "::group::$file"
+ if ../scripts/build-${language}.sh "$file"; then
+ echo "::endgroup::"
  else
- echo "- :x: $buildpath" >> $GITHUB_STEP_SUMMARY
- fi 
+ echo "::endgroup::"
+ echo "::error::Build failed for $file"
+ return 1
+ fi
+ }
+
+ # Export the build_file function for use in parallel
+ export -f build_file
+
+ # Create an array to store directories to be built
+ apps_to_build=()
+
+ files=(${{ steps.changed-files.outputs.all_modified_files }})
+
+ # Check the directories of each changed file for cdk.json
+ for file in "${files[@]}"; do
+ IFS="/" read -ra path_parts <<< "$file"
+ language=${path_parts[0]}
+ dir="${path_parts[0]}/${path_parts[1]}"
+
+ # Skip files that don't belong to the current language
+ if [[ $language != ${{ matrix.language }} ]]; then
+ continue
+ fi
+
+ apps_to_build+=("$(find "$dir" -name 'cdk.json')")
+ done
+
+ # Remove duplicate projects
+ apps_to_build=($(printf "%s\n" "${apps_to_build[@]}" | sort -u))
+
+ # Print the projects to be built
+ echo "projects to build:"
+ for dir in "${apps_to_build[@]}"; do
+ echo "- $dir"
  done
- echo "::endgroup::"
+
+ # install CDK CLI from npm, so that npx can find it later
+ cd ./${{ matrix.language }}
+ npm install
+
+ # Run the build_file function in parallel for each project to be built
+ # Halt the execution if any of the build_file invocations fail
+ parallel --keep-order --halt-on-error 2 build_file ::: "${apps_to_build[@]}"

scripts/build-csharp.sh

@@ -1,23 +1,15 @@
 #!/bin/bash
 set -euxo pipefail
 scriptdir=$(cd $(dirname $0) && pwd)
+projFile=$1
 
-# install CDK CLI from npm, so that npx can find it later
-cd $scriptdir/../csharp
-npm install
+echo "=============================="
+echo "building project: $projFile"
+echo "=============================="
 
-# Find and build all CSharp projects
-for projFile in $(find $scriptdir/../csharp -name cdk.json | grep -v node_modules); do
- (
- echo "=============================="
- echo "building project: $projFile"
- echo "=============================="
+cd $scriptdir/../$(dirname $projFile)
+if [[ -f DO_NOT_AUTOTEST ]]; then exit 0; fi
 
- cd $(dirname $projFile)
- if [[ -f DO_NOT_AUTOTEST ]]; then exit 0; fi
+dotnet build src
 
- dotnet build src
-
- $scriptdir/synth.sh
- )
-done
+$scriptdir/synth.sh

scripts/build-go.sh

@@ -1,23 +1,15 @@
 #!/bin/bash
 set -euxo pipefail
 scriptdir=$(cd $(dirname $0) && pwd)
+projFile=$1
 
-# install CDK CLI from npm, so that npx can find it later
-cd $scriptdir/../go
-npm install
+echo "=============================="
+echo "building project: $projFile"
+echo "=============================="
 
-# Find and build all Go projects
-for projFile in $(find $scriptdir/../go -name cdk.json | grep -v node_modules); do
- (
- echo "=============================="
- echo "building project: $projFile"
- echo "=============================="
+cd $scriptdir/../$(dirname $projFile)
+if [[ -f DO_NOT_AUTOTEST ]]; then exit 0; fi
 
- cd $(dirname $projFile)
- if [[ -f DO_NOT_AUTOTEST ]]; then exit 0; fi
+go get -d -t && go build
 
- go get -d -t && go build
-
- $scriptdir/synth.sh
- )
-done
+$scriptdir/synth.sh

scripts/build-java.sh

@@ -1,23 +1,15 @@
 #!/bin/bash
 set -euxo pipefail
 scriptdir=$(cd $(dirname $0) && pwd)
+projFile=$1
 
-# install CDK CLI from npm, so that npx can find it later
-cd $scriptdir/../java
-npm install
+echo "=============================="
+echo "building project: $(dirname $projFile)"
+echo "=============================="
 
-# Find and build all Maven projects
-for pomFile in $(find $scriptdir/../java -name pom.xml | grep -v node_modules); do
- (
- echo "=============================="
- echo "building project: $(dirname $pomFile)"
- echo "=============================="
+cd $scriptdir/../$(dirname $projFile)
+if [[ -f DO_NOT_AUTOTEST ]]; then exit 0; fi
 
- cd $(dirname $pomFile)
- if [[ -f DO_NOT_AUTOTEST ]]; then exit 0; fi
+mvn -q compile test
 
- mvn -q compile test
-
- $scriptdir/synth.sh
- )
-done
+$scriptdir/synth.sh

scripts/build-python.sh

@@ -1,33 +1,21 @@
 #!/bin/bash
 set -euxo pipefail
 scriptdir=$(cd $(dirname $0) && pwd)
+projFile=$1
 
+echo "=============================="
+echo "building project: $projFile"
+echo "=============================="
 
-# install CDK CLI from npm, so that npx can find it later
-cd $scriptdir/../python
-npm install
+cd $scriptdir/../$(dirname $projFile)
+[[ ! -f DO_NOT_AUTOTEST ]] || exit 0
 
-# Find and build all Python projects
-for requirements in $(find $scriptdir/../python -name requirements.txt -not -path "$scriptdir/../python/node_modules/*"); do
- (
- echo "::group::$requirements"
- echo "=============================="
- echo "building project: $requirements"
- echo "=============================="
+python3 -m venv .venv
 
- cd $(dirname $requirements)
- echo "$(tput bold)Building project at $(dirname $requirements)$(tput sgr0)"
- [[ ! -f DO_NOT_AUTOTEST ]] || exit 0
+source .venv/bin/activate
+pip install -r requirements.txt
 
- python3 -m venv /tmp/.venv
-
- source /tmp/.venv/bin/activate
- pip install -r requirements.txt
-
- $scriptdir/synth.sh
- # It is critical that we clean up the pip venv before we build the next python project
- # Otherwise, if anything gets pinned in a requirements.txt, you end up with a weird broken environment
- rm -rf /tmp/.venv
- echo "::endgroup::"
- )
-done
+$scriptdir/synth.sh
+# It is critical that we clean up the pip venv before we build the next python project
+# Otherwise, if anything gets pinned in a requirements.txt, you end up with a weird broken environment
+rm -rf .venv