Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

parallel netcdf writes, inpost with no thread #43

Merged
merged 11 commits into from
Feb 4, 2020
Merged

parallel netcdf writes, inpost with no thread #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
5994210
update parallel netcdf module files on dell
junwang-noaa Jan 29, 2020
4ffeb59
update fv3 to netcdf_parallel branch
junwang-noaa Jan 29, 2020
4c33911
update module files for hera, dell and conf file for cray
junwang-noaa Jan 31, 2020
c6a6aa8
update post lib in cray module file
junwang-noaa Jan 31, 2020
1e7225d
fix netlib on hera
junwang-noaa Jan 31, 2020
eb58daa
Update FV3 submodule
DusanJovic-NOAA Feb 3, 2020
d13dcf9
Update baselines to develop-20200202
DusanJovic-NOAA Feb 3, 2020
9ea77db
Regtest logs on wcoss_dell_p3
DusanJovic-NOAA Feb 3, 2020
dac9602
Regtest logs on wcoss_cray
DusanJovic-NOAA Feb 3, 2020
e860f7f
Regtest logs on hera
DusanJovic-NOAA Feb 3, 2020
cfdbcab
update FV3 to point to NOAA-EMC develop branch
Feb 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion FV3
Open in desktop
Submodule FV3 updated 7 files
+39 −6 fv3_cap.F90
+5 −0 io/CMakeLists.txt
+4 −1 io/makefile
+2 −1 io/module_fv3_io_def.F90
+59 −47 io/module_write_netcdf.F90
+617 −0 io/module_write_netcdf_parallel.F90
+79 −17 io/module_wrt_grid_comp.F90
4 changes: 2 additions & 2 deletions compsets/hera.input
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ platform hera.intel {
# INPUTS is the input directory, which should contain fix and parm
# files, plus any restarts or other inputs.

BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
INPUTS="@[BASELINE]"

default_resources={
Expand Down
16 changes: 8 additions & 8 deletions compsets/wcoss.input
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ platform wcoss.phase2 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'
MACHINE_ID='wcoss.phase2'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"

default_resources={
TASKS=156
Expand Down Expand Up @@ -126,8 +126,8 @@ platform wcoss.phase1 {
C768_THRD=1

MACHINE_ID='wcoss.phase1'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"

# WCOSS Phase 2 has been slow of late.
DEFAULT_TEST_WALLTIME=2700
Expand Down Expand Up @@ -213,8 +213,8 @@ platform wcoss_dell_p3 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'

BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202/"

default_resources={
TASKS=156
Expand Down Expand Up @@ -314,8 +314,8 @@ platform wcoss.cray {
BUILD_WALLTIME="3600"
DEFAULT_TEST_WALLTIME=1800

BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"

execution_time_modules=[[[
module load alps
Expand Down
2 changes: 1 addition & 1 deletion conf/configure.fv3.hera.intel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
NETCDF_LIB = -L$(NETCDF)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
Expand Down
2 changes: 1 addition & 1 deletion conf/configure.fv3.wcoss_cray
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ CFLAGS := $(INCLUDE)
FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_PARALLEL_NETCDF
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Parallel netCDF not supported on Cray ? It is fine if it is the case. The global workflow needs to have different I/O settings on different platforms.


ifeq ($(HYDRO),Y)
CPPDEFS +=
Expand Down
1,181 changes: 602 additions & 579 deletions log/report-hera.intel-log/build_fv3.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,178 changes: 593 additions & 585 deletions log/report-hera.intel-log/build_fv3_32bit.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,180 changes: 594 additions & 586 deletions log/report-hera.intel-log/build_fv3_32bit_debug.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,072 changes: 540 additions & 532 deletions log/report-hera.intel-log/build_fv3_appbuilder.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,355 changes: 686 additions & 669 deletions log/report-hera.intel-log/build_fv3_ww3.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

339 changes: 169 additions & 170 deletions log/report-hera.intel-log/rtreport.txt
View file
Open in desktop

Large diffs are not rendered by default.

1,212 changes: 607 additions & 605 deletions log/report-wcoss.cray-log/build_fv3.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,214 changes: 608 additions & 606 deletions log/report-wcoss.cray-log/build_fv3_32bit.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,092 changes: 547 additions & 545 deletions log/report-wcoss.cray-log/build_fv3_appbuilder.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

2,045 changes: 19 additions & 2,026 deletions log/report-wcoss.cray-log/build_fv3_ww3.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

307 changes: 151 additions & 156 deletions log/report-wcoss.cray-log/rtreport.txt
View file
Open in desktop

Large diffs are not rendered by default.

1,200 changes: 601 additions & 599 deletions log/report-wcoss_dell_p3-log/build_fv3.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,198 changes: 600 additions & 598 deletions log/report-wcoss_dell_p3-log/build_fv3_32bit.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,094 changes: 548 additions & 546 deletions log/report-wcoss_dell_p3-log/build_fv3_appbuilder.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

1,387 changes: 697 additions & 690 deletions log/report-wcoss_dell_p3-log/build_fv3_ww3.exe.log
View file
Open in desktop

Large diffs are not rendered by default.

314 changes: 158 additions & 156 deletions log/report-wcoss_dell_p3-log/rtreport.txt
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 4 additions & 3 deletions modulefiles/hera.intel/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ module load contrib sutils
##
module load intel/18.0.5.274
module load impi/2018.0.4
module load netcdf/4.7.0

##
## NCEP libraries (temporary version to match the CCPP requirements)
Expand All @@ -42,14 +41,16 @@ module load jasper/1.900.1
module load png/1.2.44
module load z/1.2.11
## load modules for nceppost grib
module load post/8.0.1
module load post/8.0.4

##
## load ESMF library for above compiler / MPI combination
## use pre-compiled EMSF library for above compiler / MPI combination
##
module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
module load esmf/8.0.0
module load hdf5_parallel/1.10.6
module load netcdf_parallel/4.7.4
module load esmf/8.0.0_ParallelNetCDF

##
## load cmake
Expand Down
2 changes: 1 addition & 1 deletion modulefiles/wcoss_cray/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ module load w3emc/2.3.1
module load nemsio/2.2.4

#post lib
module load post/8.0.1
module load post/8.0.4

## WCOSS Cray execution prereqs:
module load rca
Expand Down
8 changes: 4 additions & 4 deletions modulefiles/wcoss_dell_p3/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ module-whatis "loads NEMS FV3 prerequisites on Venus and Mars"
module load ips/18.0.1.163
module load impi/18.0.1
module load lsf/10.1
module load NetCDF/4.5.0
module load HDF5-serial/1.10.1
module load cmake/3.10.0
module load lsf/10.1

Expand All @@ -42,10 +40,12 @@ module load libpng/1.2.59
module load zlib/1.2.11

# load post lib
module load post/8.0.1
module load post/8.0.4

module use /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/soft/modulefiles
module load esmf/8.0.0
module load hdf5_parallel/1.10.6
module load netcdf_parallel/4.7.4
module load esmf/8.0.0_ParallelNetCDF

##
## load cmake
Expand Down
Loading