Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/stampede #213

Merged
merged 18 commits into from
Oct 20, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion MOM6-interface/mkmf
Submodule mkmf updated 1 files
+285 −0 templates/stampede-intel.mk
2 changes: 1 addition & 1 deletion WW3
188 changes: 188 additions & 0 deletions conf/configure.fv3.stampede.intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
## NEMS configuration file
##
## Platform: Orion
## Compiler: Intel with IntelMPI

SHELL=/bin/sh

################################################################################
## Include the common configuration parts

ifdef InNemsMakefile
include $(TOP)/conf/configure.nems.NUOPC
endif

############
# commands #
############
FC = mpiifort
CC = mpiicc
CXX = mpiicpc
LD = mpiifort -mkl=sequential

#########
# flags #
#########
# default is 64-bit OpenMP non-hydrostatic build using AVX2
DEBUG =
REPRO =
VERBOSE =
OPENMP = Y
AVX2 = Y
HYDRO = N
CCPP = N
STATIC = N
QUAD_PRECISION = Y
MULTI_GASES = N

include $(ESMFMKFILE)
ESMF_INC = $(ESMF_F90COMPILEPATHS)

NEMSIOINC = -I$(NEMSIO_INC)
NCEPLIBS = $(POST_LIB) $(NEMSIO_LIB) $(G2_LIB4) $(G2TMPL_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) $(CRTM_LIB) $(JASPER_ROOT)/lib64/libjasper.a -lpng -lz

##############################################
# Need to use at least GNU Make version 3.81 #
##############################################
need := 3.81
ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
ifneq ($(need),$(ok))
$(error Need at least make version $(need). Load module gmake/3.81)
endif

NETCDF_ROOT = $(NETCDF)
INCLUDE = -I$(NETCDF_ROOT)/include
NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB = -L$(NETCDF)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
CFLAGS := $(INCLUDE)

FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -save-temps -ftz -assume byterecl -nowarn -sox -align array64byte

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML

ifeq ($(HYDRO),Y)
CPPDEFS +=
else
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS += -i4 -real-size 32
else
ifeq ($(REPRO),Y)
FFLAGS += -i4 -real-size 64
else
FFLAGS += -i4 -real-size 64 -no-prec-div -no-prec-sqrt
endif
endif

ifeq ($(REPRO),Y)
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
else
ifeq ($(AVX2),Y)
# Don't use the AVX512 flags yet on hera
#FFLAGS += -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align
#CFLAGS += -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align
FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
else
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
endif
endif

ifeq ($(MULTI_GASES),Y)
CPPDEFS += -DMULTI_GASES
endif

ifeq ($(QUAD_PRECISION),Y)
CPPDEFS += -DENABLE_QUAD_PRECISION
endif

FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv -link_mpi=dbg

TRANSCENDENTALS := -fast-transcendentals
FFLAGS_OPENMP = -qopenmp
FFLAGS_VERBOSE = -v -V -what

CFLAGS += -D__IFC -sox -fp-model source

CFLAGS_OPT = -O2 -debug minimal
CFLAGS_REPRO = -O2 -debug minimal
CFLAGS_OPENMP = -qopenmp
CFLAGS_DEBUG = -O0 -g -ftrapuv -traceback

# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT
# *_TEST will match the production if no new option(s) is(are) to be tested.
FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits
CFLAGS_TEST = -O2

LDFLAGS :=
LDFLAGS_OPENMP := -qopenmp
LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M

# start with blank LIBS
LIBS :=

ifeq ($(REPRO),Y)
CPPDEFS += -DREPRO
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
FAST :=
else ifeq ($(DEBUG),Y)
CPPDEFS += -DDEBUG
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
FAST :=
else ifeq ($(TEST),Y)
CFLAGS += $(CFLAGS_TEST)
FFLAGS += $(FFLAGS_TEST)
FAST :=
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
FAST := $(TRANSCENDENTALS)
endif

ifeq ($(OPENMP),Y)
CPPDEFS += -DOPENMP
CFLAGS += $(CFLAGS_OPENMP)
FFLAGS += $(FFLAGS_OPENMP)
LDFLAGS += $(LDFLAGS_OPENMP)
endif

ifeq ($(VERBOSE),Y)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifeq ($(CCPP),Y)
CPPDEFS += -DCCPP
CFLAGS += -I$(PATH_CCPP)/include
FFLAGS += -I$(PATH_CCPP)/include
LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
endif

ifeq ($(CMEPS),Y)
CPPDEFS += -DCMEPS
endif

LDFLAGS += $(LIBS)

ifdef InNemsMakefile
FFLAGS += $(ESMF_INC)
CPPFLAGS += -traditional
EXTLIBS = $(NCEPLIBS) $(ESMF_F90LINKPATHS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB)
endif
62 changes: 62 additions & 0 deletions modulefiles/stampede.intel/fv3
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#%Module######################################################################
##
## NEMS FV3 Prerequisites: Hera/Intel

proc ModulesHelp {} {
puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Hera/Intel"
}

module-whatis "loads NEMS FV3 prerequisites for Hera/Intel"

##
## load contrib environment
## load slurm utils (arbitrary.pl layout.pl)
##
#module use -a /contrib/sutils/modulefiles
#module load sutils


##
## NCEP libraries
##
module load cmake/3.16.1
setenv CMAKE_C_COMPILER mpiicc
setenv CMAKE_CXX_COMPILER mpiicpc
setenv CMAKE_Fortran_COMPILER mpiifort
#setenv CMAKE_Platform hera.intel
module use /work/07738/kgerheis/stampede2/hpc-stack/v1.0.0-beta1/modulefiles/stack
module load hpc/1.0.0-beta1
module load hpc-intel/18.0.2
module load hpc-impi/18.0.2

module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.1
module load g2tmpl/1.9.1
module load ip/3.3.3
module load nceppost/dceca26
module load nemsio/2.5.2
module load sp/2.3.3
module load w3emc/2.7.3
module load w3nco/2.4.1

module load gfsio/1.4.1
module load sfcio/1.4.1
module load sigio/2.3.2

module load gfsio/1.4.1
module load sfcio/1.4.1
module load sigio/2.3.2

module load jasper/2.0.15
module load png/1.6.35
module load zlib/1.2.11

##
## load ESMF library for above compiler / MPI combination
## use pre-compiled EMSF library for above compiler / MPI combination
##
module load hdf5/1.10.6
module load netcdf/4.7.4
module load pio/2.5.1
module load esmf/8_1_0_beta_snapshot_27
30 changes: 15 additions & 15 deletions tests/RegressionTests_hera.intel.log
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
Wed Oct 14 00:12:28 UTC 2020
Mon Oct 19 21:56:03 UTC 2020
Start Regression test


baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1stepcold_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1stepcold_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux
Checking test 001 cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -60,7 +60,7 @@ Test 001 cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux
Checking test 002 cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -116,7 +116,7 @@ Test 002 cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux
Checking test 003 cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux results ....
Comparing phyf072.tile1.nc .........OK
Comparing phyf072.tile2.nc .........OK
Expand Down Expand Up @@ -172,7 +172,7 @@ Test 003 cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_restart
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_restart
Checking test 004 cpld_fv3_ccpp_mom6_cice_cmeps_restart results ....
Comparing phyf072.tile1.nc .........OK
Comparing phyf072.tile2.nc .........OK
Expand Down Expand Up @@ -228,7 +228,7 @@ Test 004 cpld_fv3_ccpp_mom6_cice_cmeps_restart PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads
Checking test 005 cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -284,7 +284,7 @@ Test 005 cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp
Checking test 006 cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -340,7 +340,7 @@ Test 006 cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_warm_satmedmf_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_warm_satmedmf_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf
Checking test 007 cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf results ....
Comparing phyf024.tile1.nc .........OK
Comparing phyf024.tile2.nc .........OK
Expand Down Expand Up @@ -396,7 +396,7 @@ Test 007 cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux
Checking test 008 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -452,7 +452,7 @@ Test 008 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps_frac
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps_frac/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac
Checking test 009 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -508,7 +508,7 @@ Test 009 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmrt_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmrt_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt
Checking test 010 cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt results ....
Comparing phyf024.tile1.nc .........OK
Comparing phyf024.tile2.nc .........OK
Expand Down Expand Up @@ -601,7 +601,7 @@ Test 010 cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmwav_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmwav_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt
Checking test 011 cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt results ....
Comparing phyf024.tile1.nc .........OK
Comparing phyf024.tile2.nc .........OK
Expand Down Expand Up @@ -660,7 +660,7 @@ Test 011 cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_6h_warm_debug_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_6h_warm_debug_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug
Checking test 012 cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug results ....
Comparing phyf006.tile1.nc .........OK
Comparing phyf006.tile2.nc .........OK
Expand Down Expand Up @@ -716,5 +716,5 @@ Test 012 cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug PASS


REGRESSION TEST WAS SUCCESSFUL
Wed Oct 14 07:24:59 UTC 2020
Elapsed time: 07h:12m:33s. Have a nice day!
Mon Oct 19 23:15:11 UTC 2020
Elapsed time: 01h:19m:09s. Have a nice day!
Loading