Avoid the effect of external environment on tasks (ufs-community#436)

* Enable SCHED_NATIVE_CMD on all systems. * Don't export variables for those that use SLURM. * Add some missing task specific modulefiles. * Try with an empty native line on cheyenne
danielabdi-noaa · Oct 28, 2022 · 3f4cfcf · 3f4cfcf
1 parent a1782a2
commit 3f4cfcf
Show file tree

Hide file tree

Showing 15 changed files with 34 additions and 104 deletions.
diff --git a/modulefiles/tasks/cheyenne/make_sfc_climo.local.lua b/modulefiles/tasks/cheyenne/make_sfc_climo.local.lua
@@ -0,0 +1 @@
+load("conda_regional_workflow")
diff --git a/modulefiles/tasks/gaea/make_sfc_climo.local.lua b/modulefiles/tasks/gaea/make_sfc_climo.local.lua
@@ -0,0 +1 @@
+load("miniconda_regional_workflow")
diff --git a/modulefiles/tasks/hera/make_sfc_climo.local.lua b/modulefiles/tasks/hera/make_sfc_climo.local.lua
@@ -0,0 +1 @@
+load("miniconda_regional_workflow")
diff --git a/modulefiles/tasks/jet/make_sfc_climo.local.lua b/modulefiles/tasks/jet/make_sfc_climo.local.lua
@@ -0,0 +1 @@
+load("miniconda_regional_workflow")
diff --git a/modulefiles/tasks/wcoss2/make_sfc_climo.local.lua b/modulefiles/tasks/wcoss2/make_sfc_climo.local.lua
@@ -0,0 +1,2 @@
+load(pathJoin("intel", os.getenv("intel_ver")))
+load(pathJoin("python", os.getenv("python_ver")))
diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml
diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh
@@ -184,10 +184,6 @@ module list
 if [ -n "${SRW_ENV:-}" ] ; then
  set +u
  conda activate ${SRW_ENV}
- if [ $machine = "gaea" ]; then
- conda deactivate
- conda activate ${SRW_ENV}
- fi
  set -u
 fi
 

diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml
@@ -11,11 +11,11 @@ platform:
  QUEUE_DEFAULT: normal
  QUEUE_FCST: normal
  QUEUE_HPSS: normal
- RUN_CMD_FCST: srun --mpi=pmi2 -n ${PE_MEMBER01}
- RUN_CMD_POST: srun --mpi=pmi2 -n $nprocs
+ RUN_CMD_FCST: srun --export=ALL --mpi=pmi2 -n ${PE_MEMBER01}
+ RUN_CMD_POST: srun --export=ALL --mpi=pmi2 -n $nprocs
  RUN_CMD_SERIAL: time
- RUN_CMD_UTILS: srun --mpi=pmi2 -n $nprocs
- SCHED_NATIVE_CMD: -M c3
+ RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs
+ SCHED_NATIVE_CMD: -M c3 --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/FV3LAM_pregen

diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml
@@ -15,10 +15,11 @@ platform:
  QUEUE_FCST: batch
  PARTITION_HPSS: service
  QUEUE_HPSS: batch
- RUN_CMD_FCST: srun
- RUN_CMD_POST: srun
+ RUN_CMD_FCST: srun --export=ALL
+ RUN_CMD_POST: srun --export=ALL
  RUN_CMD_SERIAL: time
- RUN_CMD_UTILS: srun
+ RUN_CMD_UTILS: srun --export=ALL
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/FV3LAM_pregen

diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml
@@ -15,10 +15,11 @@ platform:
  QUEUE_FCST: batch
  PARTITION_HPSS: service
  QUEUE_HPSS: batch
- RUN_CMD_FCST: srun
- RUN_CMD_POST: srun
+ RUN_CMD_FCST: srun --export=ALL
+ RUN_CMD_POST: srun --export=ALL
  RUN_CMD_SERIAL: time
- RUN_CMD_UTILS: srun
+ RUN_CMD_UTILS: srun --export=ALL
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/FV3LAM_pregen

diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml
@@ -12,8 +12,8 @@ platform:
  RUN_CMD_POST: mpiexec -np $nprocs
  RUN_CMD_SERIAL: time
  RUN_CMD_UTILS: mpiexec -np $nprocs
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
- BUILD_MOD_FN: wflow_noaacloud
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /contrib/EPIC/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /contrib/EPIC/UFS_SRW_App/develop/FV3LAM_pregen
  TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /contrib/EPIC/UFS_SRW_App/develop/dummy_FV3GFS_sys_dir

diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml
@@ -9,10 +9,11 @@ platform:
  QUEUE_FCST: workq
  PARTITION_HPSS: workq
  QUEUE_HPSS: workq
- RUN_CMD_FCST: srun -n ${PE_MEMBER01}
- RUN_CMD_POST: srun -n 1
- RUN_CMD_SERIAL: srun -n 1
- RUN_CMD_UTILS: srun -n $nprocs
+ RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01}
+ RUN_CMD_POST: srun --export=ALL -n 1
+ RUN_CMD_SERIAL: srun --export=ALL -n 1
+ RUN_CMD_UTILS: srun --export=ALL -n $nprocs
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/FV3LAM_pregen

diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml
@@ -15,10 +15,11 @@ platform:
  QUEUE_FCST: batch
  PARTITION_HPSS: service
  QUEUE_HPSS: batch
- RUN_CMD_FCST: srun -n ${PE_MEMBER01}
- RUN_CMD_POST: srun
+ RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01}
+ RUN_CMD_POST: srun --export=ALL
  RUN_CMD_SERIAL: time
- RUN_CMD_UTILS: srun
+ RUN_CMD_UTILS: srun --export=ALL
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/FV3LAM_pregen

diff --git a/ush/machine/singularity.yaml b/ush/machine/singularity.yaml
@@ -13,6 +13,7 @@ platform:
  RUN_CMD_POST: mpirun -np $nprocs --oversubscribe
  RUN_CMD_SERIAL: time
  RUN_CMD_UTILS: mpirun -np $nprocs --oversubscribe
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
 task_make_orog:
  TOPO_DIR: /contrib/global/glopara/fix/fix_orog

diff --git a/ush/machine/stampede.yaml b/ush/machine/stampede.yaml
@@ -13,6 +13,7 @@ platform:
  RUN_CMD_POST: ibrun -np $nprocs
  RUN_CMD_SERIAL: time
  RUN_CMD_UTILS: ibrun -np $nprocs
+ SCHED_NATIVE_CMD: --export=NONE
  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
  TEST_EXTRN_MDL_SOURCE_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data
  TEST_PREGEN_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/FV3LAM_pregen