Merge pull request ufs-community#105 from climbfuji/update_gsl_develo…

…p_from_develop_20211019

Update gsl/develop from develop 2021/10/19

.gitmodules

@@ -34,10 +34,6 @@
  path = CICE-interface/CICE
  url = https://github.com/NOAA-EMC/CICE
  branch = emc/develop
-[submodule "DATM"]
- path = DATM
- url = https://github.com/NOAA-EMC/NEMSdatm
- branch = develop
 [submodule "CDEPS"]
  path = CDEPS-interface/CDEPS
  url = https://github.com/NOAA-EMC/CDEPS

CMakeLists.txt

@@ -56,6 +56,7 @@ message("CMEPS ............ ${CMEPS}")
 ###############################################################################
 set(32BIT OFF CACHE BOOL "Enable 32BIT (single precision arithmetic in dycore)")
 set(AVX2 ON CACHE BOOL "Enable AVX2 instruction set")
+set(AVX OFF CACHE BOOL "Enable AVX-I instruction set")
 set(SIMDMULTIARCH OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
 set(DEBUG OFF CACHE BOOL "Enable DEBUG mode")
 set(DEBUG_LINKMPI ON CACHE BOOL "Enable linkmpi option when DEBUG mode is on")

MOM6-interface/CMakeLists.txt

@@ -31,6 +31,8 @@ target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BI
 target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
 target_link_libraries(mom6_obj PRIVATE fms
+ esmf
+ stochastic_physics
  NetCDF::NetCDF_Fortran)
 # OpenMP is disabled in MOM6
 #if(OpenMP_Fortran_FOUND)
@@ -58,6 +60,7 @@ target_include_directories(mom6 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_
 target_include_directories(mom6 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
 target_link_libraries(mom6 PUBLIC fms
+ stochastic_physics
  esmf
  NetCDF::NetCDF_Fortran)
 # OpenMP is disabled in MOM6
@@ -77,6 +80,8 @@ if(MOM6SOLO)
  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
  target_link_libraries(mom6solo PRIVATE mom6_obj
  fms
+ stochastic_physics
+ esmf
  NetCDF::NetCDF_Fortran)
  # OpenMP is disabled in MOM6
  #if(OpenMP_Fortran_FOUND)

MOM6-interface/mom6_files.cmake

@@ -254,6 +254,7 @@ list(APPEND mom6_src_files
  MOM6/config_src/external/ODA_hooks/ocean_da_core.F90
  MOM6/config_src/external/ODA_hooks/ocean_da_types.F90
  MOM6/config_src/external/ODA_hooks/write_ocean_obs.F90
+ MOM6/config_src/external/OCEAN_stochastic_phyiscs/MOM_stochastics.F90
 
  MOM6/config_src/infra/FMS1/MOM_coms_infra.F90
  MOM6/config_src/infra/FMS1/MOM_constants.F90

cmake/Intel.cmake

@@ -36,6 +36,10 @@ else()
  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,CORE-AVX2")
  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,CORE-AVX2")
  set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,CORE-AVX2")
+ elseif(AVX)
+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx-i")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx-i")
+ set(CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX-I")
  endif()
 endif()
 

cmake/configure_apps.cmake

@@ -27,6 +27,7 @@ endif()
 
 if(APP MATCHES "^(NG-GODAS|NG-GODAS-NEMSDATM)$")
  set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE)
+ set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE)
  set(FMS ON CACHE BOOL "Enable FMS" FORCE)
  set(MOM6 ON CACHE BOOL "Enable MOM6" FORCE)
  set(CICE6 ON CACHE BOOL "Enable CICE6" FORCE)

cmake/configure_s4.intel.cmake

@@ -0,0 +1,4 @@
+set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
+set(AVX2 OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
+set(AVX ON CACHE BOOL "Enable AVX2 instruction set" FORCE)
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)

cmake/configure_wcoss2.cmake

@@ -1,3 +1,4 @@
 set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
 set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
 set(DEBUG_LINKMPI OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
+set(AVX2 OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)

doc/UsersGuide/source/BuildingAndRunning.rst

@@ -315,23 +315,23 @@ can be identified via variables ``INPUT_NML``, ``NEMS_CONFIGURE`` and ``FV3_RUN`
 for example, by trying ``grep -n INPUT_NML *`` inside the tests/ and tests/tests/
 directories.
 
-.. _UsingUnitTest:
+.. _UsingOpnReqTest:
 
---------------------------
-Using the unit test script
---------------------------
-The unit test script ``utest`` in the tests/ directory can also be used to run
-tests. Given the name of a test, ``utest`` carries out a suite of test cases.
+---------------------------------------------
+Using the operational requirement test script
+---------------------------------------------
+The operational requirement test script ``opnReqTest`` in the tests/ directory can also be used to run
+tests. Given the name of a test, ``opnReqTest`` carries out a suite of test cases.
 Each test case addresses an aspect of the requirements new implementations
-should satisfy, which are shown in :numref:`Table %s <ImplementationRequirement>`.
-For the following discussions on utest, the user should note the distinction between
+should satisfy, which are shown in :numref:`Table %s <OperationalRequirement>`.
+For the following discussions on opnReqTest, the user should note the distinction between
 'test name' and 'test case': examples of test name are ``control``, ``cpld_control``
 and ``regional_control`` which are all found in the /tests/tests/ directory, whereas
 test case refers to any one of ``thr``, ``mpi``, ``dcp``, ``rst``, ``bit`` and ``dbg``.
 
-.. _ImplementationRequirement:
+.. _OperationalRequirement:
 
-.. table:: *Implementation requirements*
+.. table:: *Operational requirements*
 
  +----------+------------------------------------------------------------------------+
  | **Case** | **Description** |
@@ -349,47 +349,40 @@ test case refers to any one of ``thr``, ``mpi``, ``dcp``, ``rst``, ``bit`` and `
  | dbg | Model can be compiled and run to completion in debug mode |
  +----------+------------------------------------------------------------------------+
 
-The unit test uses the same testing framework used by the regression
+The operational requirement test uses the same testing framework used by the regression
 test, and therefore it is recommened that the user first read
 :numref:`Section %s <UsingRegressionTest>`. All the files in
 the subdirectories shown in :numref:`Table %s <RTSubDirs>` are relavant to the
-unit test except that the ``utest`` script replaces ``rt.sh`` and the
-``utest.bld`` file replaces ``rt.conf``. The /tests/utests/ directory contains
-utest-specific lower-level scripts used to set up run configurations.
+operational requirement test except that the ``opnReqTest`` script replaces ``rt.sh``.
+The /tests/opnReqTests/ directory contains
+opnReqTest-specific lower-level scripts used to set up run configurations.
 
 On `Tier-1 platforms <https://github.com/ufs-community/ ufs-weather-model/wiki
 /Regression-Test-Policy-for-Weather-Model-Platforms-and-Compilers>`_, tests can
-be run by first modifying the PSV file ``utest.bld`` to specify the build options
-and then invoking
+be run by invoking
 
 .. code-block:: console
 
- ./utest -n <test-name>
+ ./opnReqTest -n <test-name>
 
-For example, including in the ``utest.bld`` file the following line
-
-.. code-block:: console
-
- control | -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16 -D32BIT=ON
-
-and then executing ``./utest -n control`` performs all six test cases
-listed in :numref:`Table %s <ImplementationRequirement>` for ``control``
-test. At the end of the run, a log file ``UnitTests_<machine>.<compiler>.log``
+For example, ``./opnReqTest -n control`` performs all six test cases
+listed in :numref:`Table %s <OperationalRequirement>` for ``control``
+test. At the end of the run, a log file ``OpnReqTests_<machine>.<compiler>.log``
 is generated in tests/ directory, which informs the user whether each test case
 passed or failed. The user can choose to run a specific test case by invoking
 
 .. code-block:: console
 
- ./utest -n <test-name> -c <test-case>
+ ./opnReqTest -n <test-name> -c <test-case>
 
 where ``<test-case>`` is one or
 more comma-separated values selected from ``thr``, ``mpi``, ``dcp``, ``rst``,
-``bit``, ``dbg``. For example, ``./utest -n control -c thr,rst`` runs the
+``bit``, ``dbg``. For example, ``./opnReqTest -n control -c thr,rst`` runs the
 ``control`` test and checks the reproducibility of threading and restart.
-The user can see different command line options available to ``utest`` by
-executing ``./utest -h``; frequently used options are ``-e`` to use the ecFlow
+The user can see different command line options available to ``opnReqTest`` by
+executing ``./opnReqTest -h``; frequently used options are ``-e`` to use the ecFlow
 workflow manager, and ``-k`` to keep the ``$RUNDIR``. In the following,
-comparisons are made between the regression and unit tests on how they handle
+comparisons are made between the regression and operational requirement tests on how they handle
 different reproducibility tests.
 
 As discussed in :numref:`Section %s <UsingRegressionTest>`, the variables and
@@ -398,18 +391,18 @@ values used to configure model parameters and to set up initial conditions in th
 define default values; then a specific test file in the tests/tests/ subdirectory
 either overrides the default values or creates new variables if required by the test.
 The regression test treats the different test cases shown in
-:numref:`Table %s <ImplementationRequirement>` as different tests. Therefore, each
+:numref:`Table %s <OperationalRequirement>` as different tests. Therefore, each
 test case requires a test file in the tests/tests/ subdirectory; examples are
 ``control_2threads``, ``control_decomp``, ``control_restart`` and ``control_debug``,
 which are just variations of ``control`` test to check various reproducibilities.
 There are two potential issues with this approach. First, if several different
 variations of a given test were to be created and included in the ``rt.conf`` file,
 there are too many tests to run. Second, if a new test is added by the user, s/he
-will also have to create these variations. The idea behind the unit test is to
+will also have to create these variations. The idea behind the operational requirement test is to
 automatically configure and run these variations, or test cases, given a test file.
-For example, ``./utest -n control`` will run all six test cases in
-:numref:`Table %s <ImplementationRequirement>` based on a single ``control`` test file.
-Similarly, if the user adds a new test ``new_test``, then ``./utest -n new_test`` will
-run all test cases. This is done by the unit test script ``utest`` by adding a third
-stage of variable overrides, and the related scripts can be found in the tests/utests/
+For example, ``./opnReqTest -n control`` will run all six test cases in
+:numref:`Table %s <OperationalRequirement>` based on a single ``control`` test file.
+Similarly, if the user adds a new test ``new_test``, then ``./opnReqTest -n new_test`` will
+run all test cases. This is done by the operational requirement test script ``opnReqTest`` by adding a third
+stage of variable overrides, and the related scripts can be found in the tests/opnReqTests/
 directory.

doc/UsersGuide/source/InputsOutputs.rst

@@ -1140,6 +1140,10 @@ For the coupled NG_GODAS application, a sample *nems.configure* is shown below :
  mesh_atm = DATM_INPUT/cfsr_mesh.nc
  diro = "."
  logfile = atm.log
+ stop_n = 24
+ stop_option = nhours
+ stop_ymd = -999
+ write_restart_at_endofrun = .true.
  ::
 
  # OCN #

modulefiles/ufs_gaea.intel

@@ -22,7 +22,7 @@ module load cmake/3.20.1
 #needed for WW3 build
 module load gcc/8.3.0
 
-module use /lustre/f2/pdata/esrl/gsd/ufs/hpc-stack-v1.1.0/modulefiles/stack
+module use /lustre/f2/pdata/ncep_shared/hpc-stack/modulefiles/stack
 
 module load hpc/1.1.0
 module load hpc-intel/18.0.6.288

modulefiles/ufs_gaea.intel_debug

@@ -19,7 +19,7 @@ module load alps
 module use /lustre/f2/pdata/ncep_shared/cmake-3.20.1/modulefiles
 module load cmake/3.20.1
 
-module use /lustre/f2/pdata/esrl/gsd/ufs/hpc-stack-v1.1.0/modulefiles/stack
+module use /lustre/f2/pdata/ncep_shared/hpc-stack/modulefiles/stack
 
 module load hpc/1.1.0
 module load hpc-intel/18.0.6.288

modulefiles/ufs_s4.intel

@@ -1,41 +1,20 @@
-#%Module######################################################################
-##
-## NEMS FV3 Prerequisites: S4
+#%Module
 
 proc ModulesHelp {} {
- puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on S4"
+ puts stderr "\tcit - loads modules required for building and running UFS Model on S4"
 }
 
-module-whatis "loads NEMS FV3 prerequisites for S4"
+module-whatis "loads UFS Model prerequisites for S4"
 
-# NOTE: the "module purge" and loading of the module command are
-# handled by the module-setup.sh (or .csh) script.
-
-##
-## load programming environment
-## this typically includes compiler, MPI and job scheduler
-##
 module load license_intel/S4
-module load intel/18.0.3
-module load hdf/4.2.14
-module load hdf5/1.8.21
-module load netcdf4/4.6.2
+module use /data/prod/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.4
+module load hpc-impi/18.0.4
 
-##
-## load nwprod libraries
-##
-module use -a /data/prod/ncep_libs/intel/18.0.3/modulefiles
-module load bacio/v2.0.2
-module load sp/v2.0.2
-module load ip/v3.0.0
-module load w3nco/v2.0.6
-module load w3emc/v2.3.0
-module load nemsio/v2.2.3
-module load g2/v3.1.0
-module load g2tmpl/v1.5.0
-module load crtm/v2.3.0
-module load jasper/v1.900.1
-module load png/v1.2.44
-module load z/v1.2.6
+module load ufs_common
 
-module load esmf/v8.0.0
+setenv CC mpiicc
+setenv CXX mpiicpc
+setenv FC mpiifort
+setenv CMAKE_Platform s4.intel

modulefiles/ufs_s4.intel_debug

@@ -0,0 +1,21 @@
+#%Module
+
+proc ModulesHelp {} {
+ puts stderr "\tcit - loads modules required for building and running UFS Model on S4"
+}
+
+module-whatis "loads UFS Model prerequisites for S4"
+
+
+module load license_intel/S4
+module use /data/prod/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.4
+module load hpc-impi/18.0.4
+
+module load ufs_common_debug
+
+setenv CC mpiicc
+setenv CXX mpiicpc
+setenv FC mpiifort
+setenv CMAKE_Platform s4.intel

modulefiles/ufs_wcoss2

@@ -6,16 +6,18 @@ proc ModulesHelp {} {
 
 module-whatis "loads UFS Model prerequisites on Venus and Mars"
 
-module unload cpe-cray cce
-module load cpe-intel intel
+module load PrgEnv-intel/8.1.0
+module load intel/19.1.3.304
+module load craype/2.7.8
+module load cray-mpich/8.1.7
 
-module load cmake/3.17.3
+module load cmake/3.20.2
 
-module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/libs/hpc-stack/modulefiles/stack
 
-module load hpc/1.0.0-beta1
-module load hpc-intel/19.1.1.217
-module load hpc-cray-mpich/8.0.15
+module load hpc/1.2.0
+module load hpc-intel/19.1.3.304
+module load hpc-cray-mpich/8.1.7
 
 module load ufs_common
 

modulefiles/ufs_wcoss2_debug

@@ -6,16 +6,18 @@ proc ModulesHelp {} {
 
 module-whatis "loads UFS Model prerequisites on Venus and Mars"
 
-module unload cpe-cray cce
-module load cpe-intel intel
+module load PrgEnv-intel/8.1.0
+module load intel/19.1.3.304
+module load craype/2.7.8
+module load cray-mpich/8.1.7
 
-module load cmake/3.17.3
+module load cmake/3.20.2
 
-module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/libs/hpc-stack/modulefiles/stack
 
-module load hpc/1.0.0-beta1
-module load hpc-intel/19.1.1.217
-module load hpc-cray-mpich/8.0.15
+module load hpc/1.2.0
+module load hpc-intel/19.1.3.304
+module load hpc-cray-mpich/8.1.7
 
 module load ufs_common_debug