[develop] Update Jenkinsfile to run post-build functional tests

.cicd/Jenkinsfile

@@ -127,12 +127,13 @@ pipeline {
  }
 
  // Run the unittest functional tests that require an HPC platform
- stage('Functional Tests') {
+ stage('Functional UnitTests') {
  steps {
  echo "Running unittest on retrieve_data.py"
  sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"'
  }
  }
+
  // Run the unified build script; if successful create a tarball of the build and upload to S3
  stage('Build') {
  steps {
@@ -148,6 +149,14 @@ pipeline {
  }
  }
 
+ // Try a few Workflow Task scripts to make sure E2E tests can be launched in a follow-on 'Test' stage
+ stage('Functional WorkflowTaskTests') {
+ steps {
+ echo "Running simple workflow script task tests on ${env.SRW_PLATFORM} (using ${env.WORKSPACE})"
+ sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_ftest.sh"'
+ }
+ }
+
  // Run the unified test script
  stage('Test') {
  environment {
@@ -187,7 +196,7 @@ pipeline {
  sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/tests/WE2E/WE2E_tests_*yaml WE2E_summary*txt ${WORKSPACE}/tests/WE2E/log.*'
  // Remove the data sets from the experiments directory to conserve disk space
  sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf'
- s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+ s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: '*_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
  }
  }
  }

.cicd/scripts/srw_ftest.sh

@@ -7,7 +7,7 @@
 # We want this following a normal SRW build in an attempt to exercise environment setup, modules,
 # data sets, and workflow scripts, without using too much time nor account resources.
 # We hope to catch any snags that might prevent WE2E fundamental testing, which follows this gate.
-# NOTE: At this time, this script is a placeholder for functional test framework.
+# NOTE:
 # At this time, we are leaving the exercise of graphical plotting for a later stage, perhaps WE2E state.
 #
 # Required:
@@ -71,19 +71,27 @@ echo "DATA_LOCATION=${DATA_LOCATION}"
 sed "s|^task_get_extrn_ics:|task_get_extrn_ics:\n EXTRN_MDL_SOURCE_BASEDIR_ICS: ${DATA_LOCATION}/FV3GFS/grib2/2019061518|1" -i ush/config.yaml
 sed "s|^task_get_extrn_lbcs:|task_get_extrn_lbcs:\n EXTRN_MDL_SOURCE_BASEDIR_LBCS: ${DATA_LOCATION}/FV3GFS/grib2/2019061518|1" -i ush/config.yaml
 
+# Use staged data for HPSS supported machines
+sed 's|^platform:|platform:\n EXTRN_MDL_DATA_STORES: disk|g' -i ush/config.yaml
+
 # Activate the workflow environment ...
 source etc/lmod-setup.sh ${platform,,}
 module use modulefiles
 module load build_${platform,,}_${SRW_COMPILER}
 module load wflow_${platform,,}
 
 [[ ${FORGIVE_CONDA} == true ]] && set +e +u # Some platforms have incomplete python3 or conda support, but wouldn't necessarily block workflow tests
-conda activate regional_workflow
+conda activate workflow_tools
 set -e -u
 
+export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src
+
+# Adjust for strict limitation of stack size 
+sed "s|ulimit -s unlimited;|ulimit -S -s unlimited;|" -i ${workspace}/ush/machine/hera.yaml
+
 cd ${workspace}/ush
  # Consistency check ...
- ./config_utils.py -c ./config.yaml -v ./config_defaults.yaml -k "(\!rocoto\b)"
+ #./config_utils.py -c ./config.yaml -v ./config_defaults.yaml -k "(\!rocoto\b)"
  # Generate workflow files ...
  ./generate_FV3LAM_wflow.py
 cd ${workspace}
@@ -115,16 +123,24 @@ export OMP_NUM_THREADS=1
 )
 set +x
 
-echo "# Try the first few simple SRW tasks ..."
 results_file=${workspace}/functional_test_results_${SRW_PLATFORM}_${SRW_COMPILER}.txt
 rm -f ${results_file}
+
 status=0
-for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do
+
+# Limit to machines that are fully ready
+deny_machines=( gaea )
+if [[ ${deny_machines[@]} =~ ${platform,,} ]] ; then
+ echo "# Deny ${platform} - incomplete configuration." | tee -a ${results_file}
+else
+ echo "# Try ${platform} with the first few simple SRW tasks ..." | tee -a ${results_file}
+ for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do
  echo -n "./$task.sh ... "
  ./$task.sh > $task-log.txt 2>&1 && echo "COMPLETE" || echo "FAIL rc=$(( status+=$? ))"
  # stop at the first sign of trouble ...
  [[ 0 != ${status} ]] && echo "$task: FAIL" >> ${results_file} && break || echo "$task: COMPLETE" >> ${results_file}
-done
+ done
+fi
 
 # Set exit code to number of failures
 set +e