Merge pull request #173 from JeffBeck-NOAA/community_develop

Port to Jet

Externals.cfg

@@ -3,7 +3,7 @@ protocol = git
 repo_url = https://github.com/NCAR/UFS_UTILS
 # Specify either a branch name or a hash but not both.
 #branch = dtc/develop
-hash = 22d9e6ad
+hash = 67b2a000
 local_path = sorc/UFS_UTILS_develop
 required = True
 
@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/NCAR/UFS_UTILS
 # Specify either a branch name or a hash but not both.
 #branch = feature/chgres_grib2
-hash = b47bc84c
+hash = cd4dcaaa
 local_path = sorc/UFS_UTILS_chgres_grib2
 required = True
 
@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NCAR/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = dtc/develop
-hash = 4c2d541b
+hash = b02442b1
 local_path = sorc/ufs_weather_model
 required = True
 

jobs/JREGIONAL_MAKE_ICS

@@ -124,10 +124,9 @@ case "$MACHINE" in
  ;;
 #
 "JET")
-#
- { save_shell_opts; set +x; } > /dev/null 2>&1
-
- { restore_shell_opts; } > /dev/null 2>&1
+ ulimit -s unlimited
+# ulimit -a
+ APRUN="srun"
  ;;
 #
 "ODIN")

jobs/JREGIONAL_MAKE_LBCS

@@ -125,11 +125,11 @@ case "$MACHINE" in
 
 #
 "JET")
-#
- { save_shell_opts; set +x; } > /dev/null 2>&1
-
- { restore_shell_opts; } > /dev/null 2>&1
+ ulimit -s unlimited
+# ulimit -a
+ APRUN="srun"
  ;;
+
 #
 "ODIN")
 #

modulefiles/codes/jet/global_equiv_resol

@@ -0,0 +1,7 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module purge
+module load intel/18.0.5.274
+module load netcdf/4.6.1
+module load hdf5/1.10.4

modulefiles/codes/jet/mosaic_file

@@ -0,0 +1,7 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module purge
+module load intel/18.0.5.274
+module load netcdf/4.6.1
+module load hdf5/1.10.4

modulefiles/codes/jet/regional_grid

@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module purge
+module load intel/18.0.5.274
+module load netcdf/4.7.0
+module load hdf5/1.10.5
+

modulefiles/tasks/jet/get_extrn_ics

@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_ics task.
+#############################################################
+
+module purge
+
+module load hpss
+

modulefiles/tasks/jet/get_extrn_lbcs

@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_lbcs task.
+#############################################################
+
+module purge
+
+module load hpss
+

modulefiles/tasks/jet/make_grid

@@ -0,0 +1,9 @@
+#%Module#####################################################
+## Module file for make_grid task.
+#############################################################
+
+module purge
+
+module load intel/18.0.5.274
+module load netcdf/4.7.0
+module load hdf5/1.10.5

modulefiles/tasks/jet/make_ics.hardcoded

@@ -0,0 +1,30 @@
+#%Module#####################################################
+## Module file for make_ics task.
+#############################################################
+
+module load intel/18.0.5.274
+module load impi/2018.4.274
+module load szip
+module load hdf5
+module load netcdf/4.2.1.1
+
+module use /mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib/modulefiles
+module load esmflocal/ESMF_8_0_0_beta_snapshot_21
+
+module load w3nco-intel-sandybridge/2.0.6
+module load sp-intel-sandybridge/2.0.2
+module load bacio-intel-sandybridge/2.0.2
+module load sigio-intel-sandybridge/2.0.1
+module load sfcio-intel-sandybridge/1.0.0
+module load nemsio-intel-sandybridge/2.2.3
+
+export FCOMP=mpiifort
+#export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
+# for debugging
+export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl"
+
+export WGRIB2API_LIB="/apps/wgrib2/2.0.8/intel/18.0.5.274/lib/libwgrib2_api.a"
+export WGRIB2API_INC="/apps/wgrib2/2.0.8/intel/18.0.5.274/lib"
+export WGRIB2_LIB="/apps/wgrib2/2.0.8/intel/18.0.5.274/lib/libwgrib2.a"
+export WGRIB2_DIR="/apps/wgrib2/2.0.8/intel/18.0.5.274/bin"
+module load wgrib2/2.0.8

modulefiles/tasks/jet/make_ics.local

@@ -0,0 +1 @@
+module load wgrib2/2.0.8 

modulefiles/tasks/jet/make_lbcs.hardcoded

@@ -0,0 +1,30 @@
+#%Module#####################################################
+## Module file for make_ics task.
+#############################################################
+
+module load intel/18.0.5.274
+module load impi/2018.4.274
+module load szip
+module load hdf5
+module load netcdf/4.2.1.1
+
+module use /mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib/modulefiles
+module load esmflocal/ESMF_8_0_0_beta_snapshot_21
+
+module load w3nco-intel-sandybridge/2.0.6
+module load sp-intel-sandybridge/2.0.2
+module load bacio-intel-sandybridge/2.0.2
+module load sigio-intel-sandybridge/2.0.1
+module load sfcio-intel-sandybridge/1.0.0
+module load nemsio-intel-sandybridge/2.2.3
+
+export FCOMP=mpiifort
+#export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl"
+# for debugging
+export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl"
+
+export WGRIB2API_LIB="/apps/wgrib2/2.0.8/intel/18.0.5.274/lib/libwgrib2_api.a"
+export WGRIB2API_INC="/apps/wgrib2/2.0.8/intel/18.0.5.274/lib"
+export WGRIB2_LIB="/apps/wgrib2/2.0.8/intel/18.0.5.274/lib/libwgrib2.a"
+export WGRIB2_DIR="/apps/wgrib2/2.0.8/intel/18.0.5.274/bin"
+module load wgrib2/2.0.8

modulefiles/tasks/jet/make_lbcs.local

@@ -0,0 +1 @@
+module load wgrib2/2.0.8 

modulefiles/tasks/jet/make_orog.hardcoded

@@ -0,0 +1,9 @@
+#%Module#####################################################
+## Module file for make_orog task.
+#############################################################
+
+module purge
+# Loading Intel Compiler Suite
+module load intel/18.0.5.274
+module load netcdf/4.7.0
+module load hdf5/1.10.5

modulefiles/tasks/jet/make_sfc_climo.hardcoded

@@ -0,0 +1,12 @@
+#%Module#####################################################
+## Module file for make_sfc_climo task.
+#############################################################
+
+module purge
+
+module load intel/18.0.5.274
+module load impi/2018.0.4
+
+module use -a /scratch1/NCEPDEV/global/gwv/l819/lib/modulefiles
+module load esmflocal/8_0_48b.netcdf47
+

modulefiles/tasks/jet/run_fcst_no_ccpp.hardcoded

@@ -0,0 +1,12 @@
+#%Module#####################################################
+## Module file for run_fcst task without CCPP.
+#############################################################
+
+module purge
+
+module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles 
+module load intel/18.0.5.274
+module load impi/2018.0.4
+module load netcdf/4.6.1
+module load pnetcdf/1.10.0
+

scripts/exregional_make_grid.sh

@@ -156,25 +156,9 @@ case $MACHINE in
 #
 
 "JET")
-#
- { save_shell_opts; set +x; } > /dev/null 2>&1
-
- . /apps/lmod/lmod/init/sh
- module purge
- module load newdefaults
- module load intel/15.0.3.187
- module load impi/5.1.1.109
- module load szip
- module load hdf5
- module load netcdf4/4.2.1.1
- module list
-
- { restore_shell_opts; } > /dev/null 2>&1
 
  export APRUN="time"
- export topo_dir="/lfs3/projects/hpc-wof1/ywang/regional_fv3/fix/fix_orog"
-# . $USHDIR/set_stack_limit_jet.sh
- ulimit -a
+ topo_dir="/lfs3/projects/hpc-wof1/ywang/regional_fv3/fix/fix_orog"
  ;;
 
 

scripts/exregional_make_ics.sh

@@ -361,8 +361,13 @@ HRRRX grib2 files created after about \"${cdate_min_HRRRX}\"..."
  numsoil_out="9"
  fi
  fi
-
- geogrid_file_input_grid="/scratch2/BMC/det/beck/FV3-SAR/geo_em.d01.nc_HRRRX" # Maybe make this a fix file?
+
+ if [ "${MACHINE}" = "HERA" ]; then
+ geogrid_file_input_grid="/scratch2/BMC/det/beck/FV3-SAR/geo_em.d01.nc_HRRRX" # Maybe make this a fix file?
+ elif [ "${MACHINE}" = "JET" ]; then
+ geogrid_file_input_grid="/misc/whome/rtrr/HRRR/static/WPS/geo_em.d01.nc"
+ fi
+
  replace_vgtyp=".false."
  replace_sotyp=".false."
  replace_vgfrc=".false."
@@ -380,7 +385,7 @@ HRRRX grib2 files created after about \"${cdate_min_HRRRX}\"..."
  internal_GSD=".false."
 
  if [ "${USE_CCPP}" = "TRUE" ]; then
-  if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \
+ if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \
  [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \
  [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR_v1" ] || \
  [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \
@@ -392,7 +397,12 @@ HRRRX grib2 files created after about \"${cdate_min_HRRRX}\"..."
  fi
  fi
 
- geogrid_file_input_grid="/scratch2/BMC/det/beck/FV3-SAR/geo_em.d01.nc_RAPX" # Maybe make this a fix file?
+ if [ "${MACHINE}" = "HERA" ]; then
+ geogrid_file_input_grid="/scratch2/BMC/det/beck/FV3-SAR/geo_em.d01.nc_RAPX" # Maybe make this a fix file?
+ elif [ "${MACHINE}" = "JET" ]; then
+ geogrid_file_input_grid="/misc/whome/rtrr/HRRR/static/WPS/geo_em.d01.nc"
+ fi
+
  replace_vgtyp=".false."
  replace_sotyp=".false."
  replace_vgfrc=".false."

scripts/exregional_make_orog.sh

@@ -185,25 +185,10 @@ case $MACHINE in
 
 
 "JET")
-#
- { save_shell_opts; set +x; } > /dev/null 2>&1
-
- . /apps/lmod/lmod/init/sh
- module purge
- module load newdefaults
- module load intel/15.0.3.187
- module load impi/5.1.1.109
- module load szip
- module load hdf5
- module load netcdf4/4.2.1.1
- module list
-
- { restore_shell_opts; } > /dev/null 2>&1
-
+ ulimit -s unlimited
+ ulimit -a
  export APRUN="time"
  export topo_dir="/lfs3/projects/hpc-wof1/ywang/regional_fv3/fix/fix_orog"
-# . $USHDIR/set_stack_limit_jet.sh
- ulimit -a
  ;;
 
 

scripts/exregional_make_sfc_climo.sh

@@ -149,6 +149,10 @@ case $MACHINE in
  APRUN="srun"
  ;;
 
+"JET")
+ APRUN="srun"
+ ;;
+
 *)
  print_err_msg_exit "\
 Run command has not been specified for this machine:
@@ -242,18 +246,6 @@ Call to function to create links to surface climatology files failed."
 #
 #-----------------------------------------------------------------------
 #
-# GSK 20190430:
-# This is to make rocoto aware that the make_sfc_climo task has completed
-# (so that other tasks can be launched). This should be done through 
-# rocoto's dependencies, but not sure how to do it yet.
-#
-#-----------------------------------------------------------------------
-#
-cd_vrfy $EXPTDIR
-touch "make_sfc_climo_files_task_complete.txt"
-#
-#-----------------------------------------------------------------------
-#
 # Print message indicating successful completion of script.
 #
 #-----------------------------------------------------------------------

scripts/exregional_run_fcst.sh

@@ -150,22 +150,10 @@ case $MACHINE in
  ;;
 #
 "JET")
-#
- . /apps/lmod/lmod/init/sh
- module purge
- module load intel/15.0.3.187
- module load impi/5.1.1.109
- module load szip
- module load hdf5
- module load netcdf4/4.2.1.1
- module load contrib wrap-mpi
- module list
-
-# . $USHDIR/set_stack_limit_jet.sh
  ulimit -s unlimited
  ulimit -a
- np=${SLURM_NTASKS}
- APRUN="mpirun -np ${np}"
+ APRUN="srun"
+ LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
  ;;
 #
 "ODIN")