[develop] Add restart capability to forecast task (ufs-community#682)

* Add a flag DO_FCST_RESTART turning on/off the restart option of the forecast task. * Add a python script update_restart_input_nml_file.py replacing the six parameters related to the restart option in the FV3 input.nml file. * Add a parameter fhrot to model_configure. * Update the configuration files model_configure and nems.configure with the latest format. * This capability is the NCO's requirement. --------- Co-authored-by: chan-hoo <[email protected]> Co-authored-by: Christina.Holt <[email protected]>
danielabdi-noaa · Apr 4, 2023 · ff9eddd · ff9eddd
1 parent 36084eb
commit ff9eddd
Show file tree

Hide file tree

Showing 35 changed files with 464 additions and 154 deletions.
diff --git a/Externals.cfg b/Externals.cfg
@@ -58,7 +58,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = b066931
+hash = 8e084f9
 local_path = sorc/AQM-utils
 required = True
 

diff --git a/jobs/JREGIONAL_BIAS_CORRECTION_O3 b/jobs/JREGIONAL_BIAS_CORRECTION_O3
@@ -65,10 +65,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
 export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
 
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8` # 1 year back
-# The earliest day of training data is August 1 2020
-if [ ${TMP_STDAY} -le 20220810 ]; then 
- TMP_STDAY=20220810 
-fi
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
 #-----------------------------------------------------------------------

diff --git a/jobs/JREGIONAL_BIAS_CORRECTION_PM25 b/jobs/JREGIONAL_BIAS_CORRECTION_PM25
@@ -65,10 +65,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
 export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
 
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8` # 1 year back
-# The earliest day of training data is August 1 2020
-if [ ${TMP_STDAY} -le 20220810 ]; then 
- TMP_STDAY=20220810 
-fi
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
 #-----------------------------------------------------------------------

diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml
@@ -315,11 +315,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
  gfs_physics_nml:
  avg_max_length: 3600.0
  cdmbgwd: [0.88, 0.04]
+ debug: True
  do_deep: False
+ do_gsl_drag_ls_bl: False
+ do_gsl_drag_ss: True
+ do_gsl_drag_tofd: True
  do_mynnsfclay: True
+ do_tofd: False
  do_ugwp: False
+ do_ugwp_v0: False
+ do_ugwp_v0_nst_only: False
+ do_ugwp_v0_orog_only: False
  fhswr: 900.0
  fhlwr: 900.0
+ gwd_opt: 2
  iaer: 1011
  iccn: 2
  icliq_sw: 2
@@ -337,16 +346,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
  iopt_snf: 4
  iopt_stc: 1
  iopt_tbot: 2
- iopt_trs: 2
+ iopt_trs: !!python/none
  iovr: 3
  ldiag_ugwp: False
  lgfdlmprad: False
  lsm: 1
+ lsoil: !!python/none
  lsoil_lsm: !!python/none
  ltaerosol: False 
+ print_diff_pgr: True
+ sfclay_compute_flux: !!python/none
  xkzminv: 0.3
  xkzm_m: 1.0
  xkzm_h: 1.0
+ surf_map_nml: !!python/none
 
 FV3_GFS_v16:
  cires_ugwp_nml:
@@ -356,6 +369,8 @@ FV3_GFS_v16:
  agrid_vel_rst: False
  d2_bg_k1: 0.2
  d2_bg_k2: 0.0
+ delt_max: 0.002
+ dz_min: 6
  fv_sg_adj: 450
  hord_dp: -5
  hord_mt: 5
@@ -395,7 +410,6 @@ FV3_GFS_v16:
  lndp_type: !!python/none
  lsoil: 4
  n_var_lndp: !!python/none
- nstf_name: [2, 1, 0, 0, 0]
  prautco: [0.00015, 0.00015]
  psautco: [0.0008, 0.0005]
  satmedmf: True
@@ -506,7 +520,6 @@ FV3_GFS_v17_p8:
  min_lakeice: 0.15
  min_seaice: 0.15
  nsfullradar_diag: !!python/none
- nstf_name: [2, 0, 0, 0, 0]
  prautco: [0.00015, 0.00015]
  psautco: [0.0008, 0.0005]
  qdiag3d: False

diff --git a/parm/model_configure b/parm/model_configure
@@ -1,23 +1,26 @@
-total_member: 1
-PE_MEMBER01: {{ PE_MEMBER01 }}
 start_year: {{ start_year }}
 start_month: {{ start_month }}
 start_day: {{ start_day }}
 start_hour: {{ start_hour }}
 start_minute: 0
 start_second: 0
 nhours_fcst: {{ nhours_fcst }}
+fhrot: {{ fhrot }}
 RUN_CONTINUE: .false.
 ENS_SPS: .false.
 dt_atmos: {{ dt_atmos }}
 calendar: 'julian'
 memuse_verbose: .false.
-atmos_nthreads: {{ atmos_nthreads }}
 restart_interval: {{ restart_interval }}
 output_1st_tstep_rst: .false.
 write_dopost: {{ write_dopost }}
 ideflate: 0
 nbits: 0
+ichunk2d: -1
+jchunk2d: -1
+ichunk3d: -1
+jchunk3d: -1
+kchunk3d: -1
 quilting: {{ quilting }}
 {% if quilting %}
 #

diff --git a/parm/nems.configure b/parm/nems.configure
@@ -2,10 +2,13 @@
 #### NEMS Run-Time Configuration File #####
 #############################################
 
-{% if not print_esmf %}
 # ESMF #
+{%- if print_esmf %}
+logKindFlag: ESMF_LOGKIND_MULTI
+{%- else %}
 logKindFlag: ESMF_LOGKIND_MULTI_ON_ERROR
-{% endif %}
+{%- endif %}
+globalResourceControl: true
 
 {% if cpl_aqm %}
 # EARTH #
@@ -40,9 +43,23 @@ runSeq::
  @
 ::
 {% else %}
+# EARTH #
 EARTH_component_list: ATM
- ATM_model: fv3
- runSeq::
- ATM
- ::
+EARTH_attributes::
+ Verbosity = 0
+::
+
+# ATM #
+ATM_model: fv3
+ATM_petlist_bounds: 0 {{ pe_member01_m1 }}
+ATM_omp_num_threads: {{ atm_omp_num_threads }}
+ATM_attributes::
+ Verbosity = 0
+ Diagnostic = 0
+::
+
+# Run Sequence #
+runSeq::
+ ATM
+::
 {% endif %}
diff --git a/parm/wflow/aqm_all.yaml b/parm/wflow/aqm_all.yaml
@@ -12,7 +12,6 @@ default_aqm_task: &default_aqm
  subcyc: !cycstr "@M"
  LOGDIR: !cycstr "&LOGDIR;"
  SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;'
- memory: 2G
  native: '{{ platform.SCHED_NATIVE_CMD }}'
  nnodes: 1
  nodes: '{{ nnodes }}:ppn={{ ppn }}'
@@ -27,6 +26,7 @@ task_nexus_gfs_sfc:
  command: '&LOAD_MODULES_RUN_TASK_FP; "nexus_gfs_sfc" "&JOBSdir;/JREGIONAL_NEXUS_GFS_SFC"'
  partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
  join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+ memory: 2G
  dependency:
  or:
  and:
@@ -72,6 +72,7 @@ task_fire_emission:
  <<: *default_aqm
  command: '&LOAD_MODULES_RUN_TASK_FP; "fire_emission" "&JOBSdir;/JREGIONAL_FIRE_EMISSION"'
  join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+ memory: 2G
 
 task_point_source:
  <<: *default_aqm
@@ -146,7 +147,7 @@ task_aqm_lbcs:
  dependency:
  taskdep:
  attrs:
- task: make_lbcs
+ task: make_lbcs_mem000
 
 task_pre_post_stat:
  <<: *default_aqm

diff --git a/parm/wflow/default_workflow.yaml b/parm/wflow/default_workflow.yaml
@@ -27,6 +27,7 @@ rocoto:
  SCRIPTSdir: '{{ user.SCRIPTSdir }}'
  SLASH_ENSMEM_SUBDIR: '{% if global.DO_ENSEMBLE %}{{ "/mem#mem#" }}{% else %}{{ "/" }}{% endif %}'
  USHdir: '{{ user.USHdir }}'
+ COLDSTART: '{{ workflow.COLDSTART }}'
  WARMSTART_CYCLE_DIR: '{{ workflow.WARMSTART_CYCLE_DIR }}'
  WORKFLOW_ID: '{{ workflow.WORKFLOW_ID }}'
  attrs:

diff --git a/parm/wflow/post.yaml b/parm/wflow/post.yaml
@@ -29,7 +29,7 @@ metatask_run_ens_post:
  metatask_run_post:
  var:
  fhr: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{{ " %03d" % h }}{% endfor %}'
- cycledef: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h <= workflow.FCST_LEN_HRS %}forecast {% else %}long_forecast {% endif %}{% endfor %}'
+ cycledef: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h <= workflow.FCST_LEN_CYCL|min %}forecast {% else %}long_forecast {% endif %}{% endfor %}'
  task_run_post_mem#mem#_f#fhr#:
  <<: *default_task
  dependency:

diff --git a/scripts/exregional_aqm_lbcs.sh b/scripts/exregional_aqm_lbcs.sh
@@ -60,18 +60,15 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS}
 #
 #-----------------------------------------------------------------------
 #
-set -x
 eval ${PRE_TASK_CMDS}
 
-nprocs=$(( NNODES_AQM_LBCS*PPN_AQM_LBCS ))
-
-if [ -z "${RUN_CMD_UTILS:-}" ] ; then
+if [ -z "${RUN_CMD_AQMLBC:-}" ] ; then
  print_err_msg_exit "\
  Run command was not set in machine file. \
- Please set RUN_CMD_UTILS for your platform"
+ Please set RUN_CMD_AQM_LBC for your platform"
 else
  print_info_msg "$VERBOSE" "
- All executables will be submitted with command \'${RUN_CMD_UTILS}\'."
+ All executables will be submitted with command \'${RUN_CMD_AQMLBC}\'."
 fi
 
 #

diff --git a/scripts/exregional_bias_correction_o3.sh b/scripts/exregional_bias_correction_o3.sh
@@ -53,6 +53,7 @@ This is the ex-script for the task that runs BIAS-CORRECTION-O3.
 export KMP_AFFINITY=${KMP_AFFINITY_BIAS_CORRECTION_O3}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_BIAS_CORRECTION_O3}
 export OMP_STACKSIZE=${OMP_STACKSIZE_BIAS_CORRECTION_O3}
+export OMP_PLACES=cores
 #
 #-----------------------------------------------------------------------
 #
@@ -84,8 +85,6 @@ rm_vrfy -rf "$DATA"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
 
-set -x
-
 yyyy=${PDY:0:4}
 yyyymm=${PDY:0:6}
 yyyy_m1=${PDYm1:0:4}
@@ -207,7 +206,7 @@ fi
 
 mkdir_vrfy -p ${DATA}/data/sites
 cp_vrfy ${PARMaqm_utils}/bias_correction/config.ozone.bias_corr_${id_domain}.${cyc}z ${DATA}
- 
+
 PREP_STEP
 eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.ozone.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code."
 POST_STEP

diff --git a/scripts/exregional_bias_correction_pm25.sh b/scripts/exregional_bias_correction_pm25.sh
@@ -53,6 +53,7 @@ This is the ex-script for the task that runs BIAS-CORRECTION-PM25.
 export KMP_AFFINITY=${KMP_AFFINITY_BIAS_CORRECTION_PM25}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_BIAS_CORRECTION_PM25}
 export OMP_STACKSIZE=${OMP_STACKSIZE_BIAS_CORRECTION_PM25}
+export OMP_PLACES=cores
 #
 #-----------------------------------------------------------------------
 #
@@ -84,8 +85,6 @@ rm_vrfy -rf $DATA
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
 
-set -x
-
 yyyy=${PDY:0:4}
 yyyymm=${PDY:0:6}
 yyyy_m1=${PDYm1:0:4}

diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh
@@ -54,7 +54,6 @@ or lateral boundary conditions for the FV3.
 #
 #-----------------------------------------------------------------------
 #
-set -x
 if [ "${ICS_OR_LBCS}" = "ICS" ]; then
  if [ ${TIME_OFFSET_HRS} -eq 0 ] ; then
  file_set="anl"

diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh
@@ -244,7 +244,8 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}"
 #
 #-----------------------------------------------------------------------
 #
-if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ] || [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v17_p8" ]; then
+suites=( "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" )
+if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then
  DATA="${DATA:-${OROG_DIR}/temp_orog_data}"
  mkdir_vrfy -p ${DATA}
  cd_vrfy ${DATA}

diff --git a/scripts/exregional_nexus_emission.sh b/scripts/exregional_nexus_emission.sh
@@ -61,19 +61,16 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_NEXUS_EMISSION}
 #
 #-----------------------------------------------------------------------
 #
-set -x
 eval ${PRE_TASK_CMDS}
 
-omp_num_threads_run_aqm="${OMP_NUM_THREADS_NEXUS_EMISSION}"
-
-if [ -z "${RUN_CMD_AQM:-}" ] ; then
+if [ -z "${RUN_CMD_NEXUS:-}" ] ; then
  print_err_msg_exit "\
  Run command was not set in machine file. \
- Please set RUN_CMD_AQM for your platform"
+ Please set RUN_CMD_NEXUS for your platform"
 else
- RUN_CMD_AQM=$(eval echo ${RUN_CMD_AQM})
+ RUN_CMD_NEXUS=$(eval echo ${RUN_CMD_NEXUS})
  print_info_msg "$VERBOSE" "
- All executables will be submitted with command \'${RUN_CMD_AQM}\'."
+ All executables will be submitted with command \'${RUN_CMD_NEXUS}\'."
 fi
 #
 #-----------------------------------------------------------------------
@@ -277,7 +274,7 @@ fi
 #-----------------------------------------------------------------------
 #
 PREP_STEP
-eval ${RUN_CMD_AQM} ${EXECdir}/nexus -c NEXUS_Config.rc -r grid_spec.nc -o NEXUS_Expt_split.nc ${REDIRECT_OUT_ERR} || \
+eval ${RUN_CMD_NEXUS} ${EXECdir}/nexus -c NEXUS_Config.rc -r grid_spec.nc -o NEXUS_Expt_split.nc ${REDIRECT_OUT_ERR} || \
 print_err_msg_exit "\
 Call to execute nexus standalone for the FV3LAM failed."
 POST_STEP

diff --git a/scripts/exregional_nexus_gfs_sfc.sh b/scripts/exregional_nexus_gfs_sfc.sh
@@ -48,7 +48,6 @@ data files from disk or HPSS.
 #
 #-----------------------------------------------------------------------
 #
-set -x
 DATA="${DATA}/tmp_GFS_SFC"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA

diff --git a/scripts/exregional_post_stat_o3.sh b/scripts/exregional_post_stat_o3.sh
@@ -47,16 +47,6 @@ This is the ex-script for the task that runs POST-STAT-O3.
 #
 #-----------------------------------------------------------------------
 #
-# Set OpenMP variables.
-#
-#-----------------------------------------------------------------------
-#
-export KMP_AFFINITY=${KMP_AFFINITY_POST_STAT_O3}
-export OMP_NUM_THREADS=${OMP_NUM_THREADS_POST_STAT_O3}
-export OMP_STACKSIZE=${OMP_STACKSIZE_POST_STAT_O3}
-#
-#-----------------------------------------------------------------------
-#
 # Set run command.
 #
 #-----------------------------------------------------------------------
@@ -83,8 +73,6 @@ fi
 DATA="${DATA}/tmp_POST_STAT_O3"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
-
-set -x
 #
 #-----------------------------------------------------------------------
 #

diff --git a/scripts/exregional_post_stat_pm25.sh b/scripts/exregional_post_stat_pm25.sh
@@ -47,16 +47,6 @@ This is the ex-script for the task that runs POST-STAT-PM25.
 #
 #-----------------------------------------------------------------------
 #
-# Set OpenMP variables.
-#
-#-----------------------------------------------------------------------
-#
-export KMP_AFFINITY=${KMP_AFFINITY_POST_STAT_PM25}
-export OMP_NUM_THREADS=${OMP_NUM_THREADS_POST_STAT_PM25}
-export OMP_STACKSIZE=${OMP_STACKSIZE_POST_STAT_PM25}
-#
-#-----------------------------------------------------------------------
-#
 # Set run command.
 #
 #-----------------------------------------------------------------------
@@ -83,9 +73,6 @@ fi
 DATA="${DATA}/tmp_POST_STAT_PM25"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
-
-set -x
-
 #
 #-----------------------------------------------------------------------
 #