Merge branch 'ufs-community:develop' into RT_bmark_gefs

CDEPS-interface/cdeps_files.cmake

@@ -56,6 +56,7 @@ list(APPEND cdeps_datm_files
 list(APPEND cdeps_dice_files
  CDEPS/dice/dice_datamode_ssmi_mod.F90
  CDEPS/dice/dice_flux_atmice_mod.F90
+ CDEPS/dice/dice_datamode_cplhist_mod.F90
  CDEPS/dice/ice_comp_nuopc.F90
 )
 

CICE-interface/CMakeLists.txt

@@ -51,7 +51,8 @@ list(APPEND lib_src_files
  ${cice_shared_files_c}
  ${icepack_files}
  ${cice_mpi_comm_files}
- ${cice_nuopc_cmeps_driver_files})
+ ${cice_nuopc_cmeps_driver_files}
+ ${cice_cdeps_inline_files})
 
 list(APPEND _cice_defs FORTRANUNDERSCORE
  coupled)
@@ -84,6 +85,11 @@ if(OpenMP_Fortran_FOUND)
  target_link_libraries(cice PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 
+# ice prescribed
+add_dependencies(cice cdeps::cdeps)
+target_compile_definitions(cice PUBLIC "DISABLE_FoX")
+target_link_libraries(cice PUBLIC cdeps::cdeps)
+
 ###############################################################################
 ### Install
 ###############################################################################

CICE-interface/cice_files.cmake

@@ -150,3 +150,22 @@ list(APPEND cice_nuopc_cmeps_driver_files
  CICE/cicecore/drivers/nuopc/cmeps/ice_shr_methods.F90
  CICE/cicecore/drivers/nuopc/cmeps/ice_mesh_mod.F90
 )
+
+#-- Using ice prescribed ifndef cesmcoupled
+list(APPEND cice_cdeps_inline_files
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_orb_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_const_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_abort_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_strconvert_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_log_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_sys_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_kind_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_nl_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_cal_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_string_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_timer_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_strdata_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_stream_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_methods_mod.F90
+ ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/dshr/dshr_mod.F90
+)

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
  message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -152,7 +152,7 @@ if(FMS)
  find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)
  if(APP MATCHES "^(HAFSW)$")
  add_library(fms ALIAS FMS::fms_r4)
- elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
+ elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|ATM_DS2S|ATM_DS2S-PCICE|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
  add_library(fms ALIAS FMS::fms_r8)
  endif()
  if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|HAFS|HAFS-ALL)$")
@@ -329,6 +329,7 @@ if(CDEPS)
  add_dependencies(ufs cdeps::cdeps)
  list(APPEND _ufs_defs_private FRONT_CDEPS_DATM=cdeps_datm_comp)
  list(APPEND _ufs_defs_private FRONT_CDEPS_DOCN=cdeps_docn_comp)
+ list(APPEND _ufs_defs_private FRONT_CDEPS_DICE=cdeps_dice_comp)
  target_link_libraries(ufs PUBLIC cdeps::cdeps)
 endif()
 

cmake/configure_apps.cmake

@@ -53,6 +53,7 @@ if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL)$")
  set(FV3 ON CACHE BOOL "Enable FV3" FORCE)
  set(MOM6 ON CACHE BOOL "Enable MOM6" FORCE)
  set(CICE6 ON CACHE BOOL "Enable CICE6" FORCE)
+ set(CDEPS ON CACHE BOOL "Enable CDEPS" FORCE)
  set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE)
  if(APP MATCHES "^(S2SW|S2SWA|S2SWAL)")
  set(WW3 ON CACHE BOOL "Enable WAVEWATCH III" FORCE)
@@ -68,6 +69,20 @@ if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL)$")
  message("${APP_MSG} mode")
 endif()
 
+if(APP MATCHES "^(ATM_DS2S|ATM_DS2S-PCICE)$")
+ set(APP_MSG "Configuring UFS app in ATM_DS2S with CDEPS data")
+ set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE)
+ set(FMS ON CACHE BOOL "Enable FMS" FORCE)
+ set(FV3 ON CACHE BOOL "Enable FV3" FORCE)
+ set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE)
+ set(CDEPS ON CACHE BOOL "Enable CDEPS" FORCE)
+ if (APP MATCHES "^(ATM_DS2S-PCICE)$")
+ set(CICE6 ON CACHE BOOL "Enable CICE6" FORCE)
+ string(CONCAT APP_MSG ${APP_MSG} " with prescribed ice sea ice")
+ endif()
+ message("${APP_MSG} mode")
+endif()
+
 if(APP MATCHES "^(HAFS|HAFSW|HAFS-MOM6|HAFS-MOM6W|HAFS-ALL)$")
  set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE)
  set(CDEPS ON CACHE BOOL "Enable CDEPS" FORCE)

cmake/configure_noaacloud.intel.cmake

@@ -0,0 +1 @@
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)

doc/UsersGuide/source/CodeOverview.rst

@@ -108,7 +108,8 @@ The umbrella repository for the UFS WM is named ``ufs-weather-model``. Under thi
  ├── CDEPS-interface
  │ └── CDEPS
  │ ├── (datm) -------- CDEPS DATM
- │ └── (docn) -------- CDEPS DOCN
+ │ ├── (docn) -------- CDEPS DOCN
+ │ └── (dice) -------- CDEPS DICE
  ├── CICE-interface
  │ └── CICE -------- CICE6 sea ice model
  │ ├── (icepack) -------- Sea ice column physics
@@ -161,4 +162,4 @@ The umbrella repository for the UFS WM is named ``ufs-weather-model``. Under thi
  │ └── fv3_conf 
  └── WW3
  └── (model) -------- WW3 model
- └── (src) -------- NUOPC WW3 caps
+ └── (src) -------- NUOPC WW3 caps

driver/UFSDriver.F90

@@ -61,6 +61,9 @@ MODULE UFSDriver
 #endif
 #ifdef FRONT_CDEPS_DOCN
  use FRONT_CDEPS_DOCN, only: DOCN_SS => SetServices
+#endif
+#ifdef FRONT_CDEPS_DICE
+ use FRONT_CDEPS_DICE, only: DICE_SS => SetServices
 #endif
  ! - Handle build time ICE options:
 #ifdef FRONT_CICE6
@@ -410,6 +413,23 @@ subroutine SetModelServices(driver, rc)
  found_comp = .true.
  end if
 #endif
+#ifdef FRONT_CDEPS_DICE
+ if (trim(model) == "dice") then
+ !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+ !TODO: once component supports threading.
+ if (ompNumThreads > 1) then
+ write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+ trim(model)
+ call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg,line=__LINE__, &
+ file=__FILE__, rcToReturn=rc)
+ return ! bail out
+ endif
+ call NUOPC_DriverAddComp(driver, trim(prefix), DICE_SS, &
+ petList=petList, comp=comp, rc=rc)
+ if (ChkErr(rc,__LINE__,u_FILE_u)) return
+ found_comp = .true.
+ end if
+#endif
 #ifdef FRONT_CICE6
  if (trim(model) == "cice6") then
  call NUOPC_DriverAddComp(driver, trim(prefix), CICE6_SS, &

modulefiles/ufs_gaea.intel.lua

@@ -5,13 +5,7 @@ help([[
 
 whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
 
-load("PrgEnv-intel/8.3.3")
-load("intel-classic/2023.1.0")
-load("cray-mpich/8.1.25")
-load("python/3.9.12")
-
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/modulefiles")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
 load(pathJoin("stack-intel", stack_intel_ver))
@@ -22,8 +16,13 @@ load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
 load(pathJoin("stack-python", stack_python_ver))
 
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
 load("ufs_common")
-load("nccmp/1.9.0.1")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
 
 unload("darshan-runtime")
 unload("cray-libsci")

tests/bl_date.conf

@@ -1 +1 @@
-export BL_DATE=20240603
+export BL_DATE=20240607

tests/default_vars.sh

@@ -419,6 +419,7 @@ export CMP_DATAONLY=false
 # Defaults for ufs.configure
 export esmf_logkind="ESMF_LOGKIND_MULTI"
 export DumpFields="false"
+export MED_history_n=1000000
 
 export_fv3_v16 ()
 {
@@ -498,9 +499,12 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export DICE_CDEPS=false
+export CICE_PRESCRIBED=false
 export CDEPS_INLINE=false
 export POSTAPP='global'
 export USE_MERRA2=.true.
+export NESTED=.false.
 
 export NTILES=6
 export INPES=${INPES_dflt}
@@ -904,6 +908,7 @@ export_cice6() {
  export DT_CICE=${DT_ATMOS}
  export CICE_NPT=999
  export CICE_RUNTYPE=initial
+ export CICE_ICE_IC='cice_model.res.nc'
  export CICE_RUNID=unknown
  export CICE_USE_RESTART_TIME=.false.
  export CICE_RESTART_DIR=./RESTART/
@@ -954,6 +959,12 @@ export_cice6() {
  export CICE_BLCKX
  export CICE_BLCKY
  export CICE_DECOMP=slenderX2
+
+ #ds2s
+ export MESH_DICE=none
+ export stream_files_dice=none
+ export CICE_PRESCRIBED=false
+ export DICE_CDEPS=false
 }
 
 # Defaults for the MOM6 model namelist, mx100
@@ -1056,6 +1067,8 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export DICE_CDEPS=false
+export CICE_PRESCRIBED=false
 export CDEPS_INLINE=false
 export FV3BMIC='p8c'
 export BMIC=.false.
@@ -1222,6 +1235,10 @@ export AOD_FRQ=060000
 export RESTART_FILE_PREFIX=''
 export RESTART_FILE_SUFFIX_SECS=''
 export RT35D=''
+
+#CDEPS ds2s
+export MESH_DICE=none
+export stream_files_dice=none
 }
 export_35d_run ()
 {

tests/fv3_conf/control_run.IN

@@ -10,14 +10,17 @@ elif [ $NPX = 385 ]; then
 elif [ $NPX = 769 ]; then
  inputdir=FV3_input_data768
 fi
-echo "inputdir=$inputdir,NPX=$NPX"
+
+echo "inputdir=$inputdir,NPX=$NPX,NESTED=$NESTED"
 
 OPNREQ_TEST=${OPNREQ_TEST:-false}
 V2_SFC_FILE=${V2_SFC_FILE:-false}
 SUFFIX=${RT_SUFFIX}
 if [ $WARM_START = .false. ]; then
  mkdir INPUT RESTART
- if [ "$V2_SFC_FILE" = "true" ]; then
+ if [ "$NESTED" = .true. ]; then
+ cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127_nested/* ./INPUT/.
+ elif [ "$V2_SFC_FILE" = "true" ]; then
  cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127_v2_sfc/* ./INPUT/.
  else
  cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127/* ./INPUT/.

tests/fv3_conf/cpld_docn_dice.IN

@@ -0,0 +1,15 @@
+mkdir -p INPUT
+
+SUFFIX=${RT_SUFFIX}
+if [[ ${OPNREQ_TEST} == true ]]; then
+ SUFFIX=${BL_SUFFIX}
+fi
+
+#CDEPS data files
+#cp @[INPUTDATA_ROOT]/DOCN_DICE_cplhist/ufs.cpld.cpl*.nc INPUT
+module load nco
+ncrcat ../${DEP_RUN}${SUFFIX}/ufs.cpld.cpl.hi.ice*nc ./INPUT/ufs.cpld.cpl.hi.ice.nc
+ncrcat ../${DEP_RUN}${SUFFIX}/ufs.cpld.cpl.hi.ocn*nc ./INPUT/ufs.cpld.cpl.hi.ocn.nc
+
+#CDEPS fix files - same for ocn/ice
+cp @[INPUTDATA_ROOT]/CICE_FIX/@[OCNRES]/mesh.mx@[OCNRES].nc INPUT

tests/fv3_conf/cpld_docn_pcice.IN

@@ -0,0 +1,4 @@
+mkdir -p INPUT
+
+#For consistency, same CDEPS docn and CICE ice prescribed input files (data and ESMF_mesh)
+cp @[INPUTDATA_ROOT]/DOCN_DICE_ERA5/* INPUT