Merge pull request ufs-community#96 from grantfirl/ufs-dev-PR56

UFS-dev PR#56

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
  message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -69,7 +69,7 @@ set(MOVING_NEST OFF CACHE BOOL "Enable moving nest code")
 set(OPENMP ON CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER OFF CACHE BOOL "Enable JEDI as top level driver")
-set(CMEPS_AOFLUX OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator") 
+set(CMEPS_AOFLUX OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -148,7 +148,7 @@ if(FMS)
  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS)$")
  add_library(fms ALIAS FMS::fms_r8)
  endif()
- if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMW|ATML|LND|HAFS|HAFS-ALL)$")
+ if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|LND|HAFS|HAFS-ALL)$")
  if(32BIT)
  add_library(fms ALIAS FMS::fms_r4)
  else()
@@ -201,7 +201,7 @@ endif()
 ### Wave components [WW3]
 ###############################################################################
 if(WW3)
- if(APP MATCHES "^(ATMW)$")
+ if(APP MATCHES "^(ATMWM)$")
  set(SWITCH "multi_esmf" CACHE STRING "ESMF cap")
  set(UFS_CAP "MULTI_ESMF" CACHE STRING "Build with MULTI_ESMF cap")
  list(APPEND _ufs_defs_private FRONT_WW3=WMESMFMD)

cmake/configure_apps.cmake

@@ -13,13 +13,17 @@
 ###############################################################################
 ### Configure Application Components
 ###############################################################################
-if(APP MATCHES "^(ATM|ATMW|ATMAQ|ATML)$")
+if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML)$")
  set(FMS ON CACHE BOOL "Enable FMS" FORCE)
  set(FV3 ON CACHE BOOL "Enable FV3" FORCE)
  set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE)
  if(APP MATCHES "ATMW")
  set(WW3 ON CACHE BOOL "Enable WAVEWATCH III" FORCE)
  message("Configuring UFS app in Atmosphere with Waves mode")
+ set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE)
+ elseif(APP MATCHES "ATMWM")
+ set(WW3 ON CACHE BOOL "Enable WAVEWATCH III" FORCE)
+ message("Configuring UFS app in Atmosphere with Waves mode")
  elseif(APP MATCHES "ATMAQ")
  set(AQM ON CACHE BOOL "Enable AQM" FORCE)
  message("Configuring UFS app in Atmosphere with Air Quality mode")

tests/default_vars.sh

@@ -13,7 +13,8 @@
 
  THRD_cpl_atmw=1
  INPES_cpl_atmw=3; JNPES_cpl_atmw=8; WPG_cpl_atmw=6
- WAV_tasks_atmw=30
+ WAV_tasks_cpl_atmw=30
+ WAV_thrds_cpl_atmw=1
 
  THRD_cpl_c48=1
  INPES_cpl_c48=1; JNPES_cpl_c48=1; WPG_cpl_c48=6
@@ -535,11 +536,11 @@ export IAU_DRYMASSFIXER=.false.
 #waves
 export WW3RSTDTHR=12
 export DT_2_RST="$(printf "%02d" $(( ${WW3RSTDTHR}*3600 )))"
-export DTRST=0
-export RSTTYPE=T
 export WW3OUTDTHR=1
 export DTFLD="$(printf "%02d" $(( ${WW3OUTDTHR}*3600 )))"
 export DTPNT="$(printf "%02d" $(( ${WW3OUTDTHR}*3600 )))"
+export DTRST=0
+export RSTTYPE=T
 export GOFILETYPE=1
 export POFILETYPE=1
 export OUTPARS_WAV="WND HS FP DP PHS PTP PDIR"
@@ -565,8 +566,15 @@ export RST_BEG=$RUN_BEG
 export RST_2_BEG=$RUN_BEG
 export RST_END=$RUN_END
 export RST_2_END=$RUN_END
+export WAV_CUR='F'
+export WAV_ICE='F'
+export WAV_IC1='F'
+export WAV_IC5='F'
 # ATMW
 export MULTIGRID=true
+export MODDEF_WAV=mod_def.glo_1deg
+export MESH_WAV=mesh.glo_1deg.nc
+
 # ATMA
 export AOD_FRQ=060000
 
@@ -815,6 +823,7 @@ export BLCKX=`expr $NX_GLB / $np2`
 export BLCKY=`expr $NY_GLB / 2`
 export MESHOCN_ICE=mesh.mx${OCNRES}.nc
 export WAVDOMAIN=mx${OCNRES}
+export MODDEF_WAV=mod_def.mx${OCNRES}
 export MESH_WAV=mesh.${WAVDOMAIN}.nc
 export CICEGRID=grid_cice_NEMS_mx${OCNRES}.nc
 export CICEMASK=kmtu_cice_NEMS_mx${OCNRES}.nc
@@ -835,23 +844,16 @@ export GRIDOCN=A
 export GRIDICE=B
 
 #wave
-export INPUT_CURFLD='C F Currents'
-export INPUT_ICEFLD='C F Ice concentrations'
 export WW3RSTDTHR=3
-export WW3OUTDTHR=3
 export DT_2_RST="$(printf "%02d" $(( ${WW3RSTDTHR}*3600 )))"
+export WW3OUTDTHR=3
 export DTFLD="$(printf "%02d" $(( ${WW3OUTDTHR}*3600 )))"
 export DTPNT="$(printf "%02d" $(( ${WW3OUTDTHR}*3600 )))"
-export WW3GRIDLINE="'ww3' 'no' 'CPL:native' 'CPL:native' 'CPL:native' 'no' 'no' 'no' 'no' 'no' 1 1 0.00 1.00 F"
-export UNIPOINTS='points'
-export RUN_BEG="${SYEAR}${SMONTH}${SDAY} $(printf "%02d" $(( ${SHOUR} )))0000"
-export RUN_END="2100${SMONTH}${SDAY} $(printf "%02d" $(( ${SHOUR} )))0000"
-export OUT_BEG=$RUN_BEG
-export OUT_END=$RUN_END
-export RST_BEG=$RUN_BEG
-export RST_2_BEG=$RUN_BEG
-export RST_END=$RUN_END
-export RST_2_END=$RUN_END
+# waves when using shel.nml.IN
+export WAV_CUR='C'
+export WAV_ICE='C'
+export WAV_IC1='F'
+export WAV_IC5='F'
 # gocart inst_aod output; uses AERO_HIST.rc.IN from parm/gocart directory
 export AOD_FRQ=060000
 
@@ -1075,13 +1077,18 @@ export DLON=0.03
 export DLAT=0.03
 
 # shel.inp
-export INPUT_CURFLD='C F Currents'
-export INPUT_ICEFLD='F F Ice concentrations'
 # input.nml
 export CPL_IMP_MRG=.true.
 
 export DIAG_TABLE=diag_table_hafs
 export FIELD_TABLE=field_table_hafs
+export WW3RSTDTHR=${FHMAX}
+export DT_2_RST="$(printf "%02d" $(( ${WW3RSTDTHR}*3600 )))"
+export WW3OUTDTHR=3
+export DTFLD="$(printf "%02d" $(( ${WW3OUTDTHR}*3600 )))"
+export DTPNT="$(printf "%02d" $(( ${WW3OUTDTHR}*3600 )))"
+export OUTPARS_WAV="WND HS T01 T02 DIR FP DP PHS PTP PDIR UST CHA USP"
+export WAV_CUR='C'
 
 # nems.configure
 export med_model=cmeps
@@ -1091,6 +1098,7 @@ export CPLMODE=hafs
 export RUNTYPE=startup
 export USE_COLDSTART=false
 export MESH_WAV=mesh.hafs.nc
+export MODDEF_WAV=mod_def.natl_6m
 export MULTIGRID=false
 }
 

tests/fv3_conf/control_run.IN

@@ -91,16 +91,21 @@ else
 fi
 
 if [ $CPLWAV = .true. ]; then
- cp @[INPUTDATA_ROOT_WW3]/mod_def.pointsatmw mod_def.points
- cp @[INPUTDATA_ROOT_WW3]/mod_def.* .
- cp @[INPUTDATA_ROOT_WW3]/rmp_src* .
+ if [ $MULTIGRID = 'true' ]; then
+ cp @[INPUTDATA_ROOT_WW3]/mod_def.pointsatmw mod_def.points
+ cp @[INPUTDATA_ROOT_WW3]/@[MODDEF_WAV] .
+ cp @[INPUTDATA_ROOT_WW3]/rmp_src* .
+ else
+ cp @[INPUTDATA_ROOT_WW3]/@[MODDEF_WAV] mod_def.ww3
+ cp @[INPUTDATA_ROOT_WW3]/@[MESH_WAV] .
+ fi
 fi
 
 #inline post
 if [[ $POSTAPP = 'global' ]]; then
- cp ${PATHRT}/parm/post_itag itag
- cp ${PATHRT}/parm/postxconfig-NT.txt postxconfig-NT.txt
- cp ${PATHRT}/parm/postxconfig-NT_FH00.txt postxconfig-NT_FH00.txt
+ cp ${PATHRT}/parm/post_itag_gfs itag
+ cp ${PATHRT}/parm/postxconfig-NT-gfs.txt postxconfig-NT.txt
+ cp ${PATHRT}/parm/postxconfig-NT-gfs_FH00.txt postxconfig-NT_FH00.txt
  cp ${PATHRT}/parm/params_grib2_tbl_new params_grib2_tbl_new
 fi
 

tests/fv3_conf/cpld_control_run.IN

@@ -50,12 +50,7 @@ cp @[INPUTDATA_ROOT]/CICE_FIX/@[OCNRES]/mesh.mx@[OCNRES].nc .
 
 # WW3 fix/input
 if [[ $CPLWAV == .true. ]]; then
- cp @[INPUTDATA_ROOT_WW3]/mod_def.points .
- if [[ $WAVDOMAIN == mx025 ]]; then
- cp @[INPUTDATA_ROOT_WW3]/mod_def.@[WAVDOMAIN]lite mod_def.ww3
- else
- cp @[INPUTDATA_ROOT_WW3]/mod_def.@[WAVDOMAIN] mod_def.ww3
- fi
+ cp @[INPUTDATA_ROOT_WW3]/@[MODDEF_WAV] mod_def.ww3
  if [[ ${WAVDOMAIN:0:2} != mx ]]; then
  cp @[INPUTDATA_ROOT_WW3]/@[MESH_WAV] .
  fi
@@ -145,9 +140,9 @@ fi
 
 #inline post
 if [ $WRITE_DOPOST = .true. ]; then
- cp ${PATHRT}/parm/post_itag itag
- cp ${PATHRT}/parm/postxconfig-NT.txt postxconfig-NT.txt
- cp ${PATHRT}/parm/postxconfig-NT_FH00.txt postxconfig-NT_FH00.txt
+ cp ${PATHRT}/parm/post_itag_gfs itag
+ cp ${PATHRT}/parm/postxconfig-NT-gfs.txt postxconfig-NT.txt
+ cp ${PATHRT}/parm/postxconfig-NT-gfs_FH00.txt postxconfig-NT_FH00.txt
  cp ${PATHRT}/parm/params_grib2_tbl_new params_grib2_tbl_new
 fi
 

tests/fv3_conf/hafs_ww3_run.IN

@@ -2,9 +2,8 @@
 if [ ${CNTL_DIR} = "hafs_regional_atm_wav" ] || \
  [ ${CNTL_DIR} = "hafs_regional_atm_ocn_wav" ] || \
  [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
- cp @[INPUTDATA_ROOT_WW3]/mod_def.natl_6m ./mod_def.ww3
- cp @[INPUTDATA_ROOT_WW3]/mod_def.points ./mod_def.points
- cp @[INPUTDATA_ROOT_WW3]/mesh.hafs.nc ./mesh.hafs.nc
+ cp @[INPUTDATA_ROOT_WW3]/@[MODDEF_WAV] ./mod_def.ww3
+ cp @[INPUTDATA_ROOT_WW3]/@[MESH_WAV] ./mesh.hafs.nc
  if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
  cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/WW3_hafs_regional_input_data/wind.natl_6m_2020082512 ./wind.ww3
  else

tests/parm/nems.configure.atmw.IN

@@ -0,0 +1,94 @@
+#############################################
+#### NEMS Run-Time Configuration File #####
+#############################################
+
+# ESMF #
+logKindFlag: ESMF_LOGKIND_MULTI
+globalResourceControl: true
+
+# EARTH #
+EARTH_component_list: MED ATM WAV
+EARTH_attributes::
+ Verbosity = 0
+::
+
+# MED #
+MED_model: @[med_model]
+MED_petlist_bounds: @[med_petlist_bounds]
+MED_omp_num_threads: @[med_omp_num_threads]
+::
+
+# ATM #
+ATM_model: @[atm_model]
+ATM_petlist_bounds: @[atm_petlist_bounds]
+ATM_omp_num_threads: @[atm_omp_num_threads]
+ATM_attributes::
+ Verbosity = 0
+ DumpFields = false
+::
+
+# WAV #
+WAV_model: @[wav_model]
+WAV_petlist_bounds: @[wav_petlist_bounds]
+WAV_omp_num_threads: @[wav_omp_num_threads]
+WAV_attributes::
+ Verbosity = 0
+ OverwriteSlice = false
+ diro = "."
+ logfile = wav.log
+ mesh_wav = @[MESH_WAV]
+ multigrid = @[MULTIGRID]
+ user_sets_restname = true
+::
+
+
+# Run Sequence #
+runSeq::
+ @@[coupling_interval_sec]
+ MED med_phases_prep_atm
+ MED med_phases_prep_wav_accum
+ MED med_phases_prep_wav_avg
+ MED -> ATM :remapMethod=redist
+ MED -> WAV :remapMethod=redist
+ ATM
+ WAV
+ ATM -> MED :remapMethod=redist
+ MED med_phases_post_atm
+ WAV -> MED :remapMethod=redist
+ MED med_phases_post_wav
+ MED med_phases_restart_write
+ @
+::
+
+# CMEPS variables
+
+DRIVER_attributes::
+::
+
+MED_attributes::
+ ATM_model = @[atm_model]
+ WAV_model = @[wav_model]
+ history_n = 1
+ history_option = nhours
+ history_ymd = -999
+ coupling_mode = @[CPLMODE]
+ history_tile_atm = @[ATMTILESIZE]
+::
+
+ALLCOMP_attributes::
+ ScalarFieldCount = 2
+ ScalarFieldIdxGridNX = 1
+ ScalarFieldIdxGridNY = 2
+ ScalarFieldName = cpl_scalars
+ start_type = @[RUNTYPE]
+ restart_dir = RESTART/
+ case_name = ufs.atmw
+ restart_n = @[RESTART_N]
+ restart_option = nhours
+ restart_ymd = -999
+ dbug_flag = @[cap_dbug_flag]
+ use_coldstart = false
+ stop_n = @[FHMAX]
+ stop_option = nhours
+ stop_ymd = -999
+::

tests/parm/post_itag → tests/parm/post_itag_gfs

@@ -2,5 +2,5 @@
  MODELNAME='GFS'
 /
 &NAMPGB
- KPO=57,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,40.,30.,20.,15.,10.,7.,5.,3.,2.,1.,0.7,0.4,0.2,0.1,0.07,0.04,0.02,0.01,
+ KPO=57,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,40.,30.,20.,15.,10.,7.,5.,3.,2.,1.,0.7,0.4,0.2,0.1,0.07,0.04,0.02,0.01,rdaod=.true.,
 /

tests/parm/postxconfig-NT.txt → tests/parm/postxconfig-NT-gfs.txt

@@ -1,6 +1,6 @@
 2
 112
-209
+210
 GFSPRS
 0
 ncep_nco
@@ -7750,6 +7750,43 @@ between_first_second_limit
 ?
 ?
 ?
+1006
+SDEN_ON_SURFACE
+?
+1
+tmpl4_0
+SDEN
+?
+?
+surface
+0
+?
+0
+?
+?
+0
+?
+0
+?
+?
+?
+0
+0.0
+0
+0.0
+?
+0
+0.0
+0
+0.0
+1
+6.0
+0
+0
+0
+?
+?
+?
 GFSFLX
 0
 ncep_nco