diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..a54061e9d0 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,235 @@ +cmake_minimum_required(VERSION 2.8.12) + +foreach(env_var IN ITEMS + CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_Fortran_COMPILER + CMAKE_Platform) + if ( NOT DEFINED ENV{${env_var}}) + message(FATAL_ERROR "${env_var} is not defined") + endif() +endforeach() + +set(CMAKE_C_COMPILER $ENV{CMAKE_C_COMPILER}) +set(CMAKE_CXX_COMPILER $ENV{CMAKE_CXX_COMPILER}) +set(CMAKE_Fortran_COMPILER $ENV{CMAKE_Fortran_COMPILER}) +set(CMAKE_Platform $ENV{CMAKE_Platform}) + +project(NEMSfv3gfs C CXX Fortran) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) +find_package(MPI REQUIRED) +find_package(ESMF REQUIRED) + +include(${PROJECT_SOURCE_DIR}/cmake/configure_${CMAKE_Platform}.cmake) + +if(NOT DEFINED PHYS) + set(PHYS gfs) +endif() +message("") +message("Selected physics package: ${PHYS}") +message("") + +add_definitions(-Duse_libMPI) +add_definitions(-Duse_netCDF) +add_definitions(-Duse_WRTCOMP) +add_definitions(-DSPMD) +add_definitions(-DUSE_LOG_DIAG_FIELD_INFO) +#add_definitions(-Duse_LARGEFILE) +add_definitions(-DUSE_GFSL63) +add_definitions(-DGFS_PHYS) +add_definitions(-DMOIST_CAPPA) +add_definitions(-DUSE_COND) +add_definitions(-DNEW_TAUCTMAX) +add_definitions(-DINTERNAL_FILE_NML) + +############################################################################### +### FMS +############################################################################### +add_library( + fms + + FMS/amip_interp/amip_interp.F90 + FMS/astronomy/astronomy.F90 + FMS/axis_utils/axis_utils.F90 + FMS/block_control/block_control.F90 + FMS/column_diagnostics/column_diagnostics.F90 + FMS/constants/constants.F90 + FMS/coupler/atmos_ocean_fluxes.F90 + FMS/coupler/coupler_types.F90 + FMS/coupler/ensemble_manager.F90 + FMS/data_override/data_override.F90 + FMS/diag_manager/diag_axis.F90 + FMS/diag_manager/diag_data.F90 + FMS/diag_manager/diag_grid.F90 + FMS/diag_manager/diag_manager.F90 + FMS/diag_manager/diag_manifest.F90 + FMS/diag_manager/diag_output.F90 + FMS/diag_manager/diag_table.F90 + FMS/diag_manager/diag_util.F90 + FMS/drifters/cloud_interpolator.F90 + FMS/drifters/drifters_comm.F90 + FMS/drifters/drifters_core.F90 + FMS/drifters/drifters.F90 + FMS/drifters/drifters_input.F90 + FMS/drifters/drifters_io.F90 + FMS/drifters/quicksort.F90 + FMS/exchange/stock_constants.F90 + FMS/exchange/xgrid.F90 + FMS/fft/fft99.F90 + FMS/fft/fft.F90 + FMS/field_manager/field_manager.F90 + FMS/field_manager/fm_util.F90 + FMS/fms/fms.F90 + FMS/fms/fms_io.F90 + FMS/horiz_interp/horiz_interp_bicubic.F90 + FMS/horiz_interp/horiz_interp_bilinear.F90 + FMS/horiz_interp/horiz_interp_conserve.F90 + FMS/horiz_interp/horiz_interp.F90 + FMS/horiz_interp/horiz_interp_spherical.F90 + FMS/horiz_interp/horiz_interp_type.F90 + FMS/interpolator/interpolator.F90 + FMS/memutils/memutils.F90 + FMS/mosaic/gradient.F90 + FMS/mosaic/grid.F90 + FMS/mosaic/mosaic.F90 + FMS/mpp/mpp_data.F90 + FMS/mpp/mpp_domains.F90 + FMS/mpp/mpp_efp.F90 + FMS/mpp/mpp.F90 + FMS/mpp/mpp_io.F90 + FMS/mpp/mpp_memutils.F90 + FMS/mpp/mpp_parameter.F90 + FMS/mpp/mpp_pset.F90 + FMS/mpp/mpp_utilities.F90 + FMS/oda_tools/oda_core_ecda.F90 + FMS/oda_tools/oda_core.F90 + FMS/oda_tools/oda_types.F90 + FMS/oda_tools/write_ocean_data.F90 + FMS/oda_tools/xbt_drop_rate_adjust.f90 + FMS/platform/platform.F90 + FMS/random_numbers/MersenneTwister.F90 + FMS/random_numbers/random_numbers.F90 + FMS/sat_vapor_pres/sat_vapor_pres.F90 + FMS/sat_vapor_pres/sat_vapor_pres_k.F90 + FMS/station_data/station_data.F90 + FMS/time_interp/time_interp_external.F90 + FMS/time_interp/time_interp.F90 + FMS/time_manager/get_cal_time.F90 + FMS/time_manager/time_manager.F90 + FMS/topography/gaussian_topog.F90 + FMS/topography/topography.F90 + FMS/tracer_manager/tracer_manager.F90 + FMS/tridiagonal/tridiagonal.F90 + FMS/memutils/memuse.c + FMS/mosaic/create_xgrid.c + FMS/mosaic/gradient_c2l.c + FMS/mosaic/interp.c + FMS/mosaic/mosaic_util.c + FMS/mosaic/read_mosaic.c + FMS/mpp/affinity.c + FMS/mpp/nsclock.c + FMS/mpp/threadloc.c +) +# stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs +set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0") + +target_include_directories(fms PUBLIC ${PROJECT_SOURCE_DIR}/FMS/include FMS/include FMS/fms FMS/mpp/include) +target_include_directories(fms PRIVATE ${NETCDF_INC_DIR}) + +set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS) +set_target_properties(fms PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod) +target_include_directories(fms PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod) + +############################################################################### +### FV3 +############################################################################### +add_subdirectory(FV3) + +############################################################################### +### WW3 +############################################################################### +if(WW3) + if(${CMAKE_Platform} STREQUAL "hera.intel") + set(WW3_COMP "hera") + else() + message(FATAL_ERROR "Platform ${CMAKE_Platform} is not supported in WW3") + endif() + message("Build WW3:") + message(" run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nems") + message(" in: ${PROJECT_SOURCE_DIR}/WW3/model/esmf") + message("") + + add_custom_target(ww3_nems + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/esmf + COMMAND ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nems > make.log 2>&1 + ) +endif() + +############################################################################### +### NEMS +############################################################################### +configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h + ${PROJECT_BINARY_DIR}/NEMS/src/ESMFVersionDefine.h + COPYONLY) +# until we remove "../ESMFVersionDefine.h" from ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90 +configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFVersionDefine_ESMF_NUOPC.h + ${PROJECT_BINARY_DIR}/NEMS/ESMFVersionDefine.h + COPYONLY) +configure_file(${PROJECT_SOURCE_DIR}/NEMS/src/ESMFConvenienceMacros.h + ${PROJECT_BINARY_DIR}/NEMS/src/ESMFConvenienceMacros.h + COPYONLY) + +add_executable( + NEMS.exe + + NEMS/src/MAIN_NEMS.F90 + NEMS/src/module_NEMS_UTILS.F90 + NEMS/src/module_MEDIATOR_methods.F90 + NEMS/src/module_MEDIATOR.F90 + NEMS/src/module_MEDIATOR_SpaceWeather.F90 + NEMS/src/module_EARTH_INTERNAL_STATE.F90 + NEMS/src/module_EARTH_GRID_COMP.F90 + NEMS/src/module_NEMS_INTERNAL_STATE.F90 + NEMS/src/module_NEMS_GRID_COMP.F90 + NEMS/src/module_NEMS_Rusage.F90 + NEMS/src/nems_c_rusage.c + NEMS/src/ENS_Cpl/ENS_CplComp_ESMFMod_STUB.F90 +) + +target_compile_definitions(NEMS.exe PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}) +target_compile_definitions(NEMS.exe PRIVATE -DFRONT_FMS) +target_compile_definitions(NEMS.exe PRIVATE -DFRONT_FV3=fv3gfs_cap_mod) + +target_include_directories(NEMS.exe PRIVATE ${ESMF_MOD} ) +target_include_directories(NEMS.exe PRIVATE ${PROJECT_BINARY_DIR}/NEMS/src) + +set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/NEMS/mod) + +if(WW3) + target_compile_definitions(NEMS.exe PRIVATE -DFRONT_WW3=WMESMFMD) + set_target_properties(NEMS.exe PROPERTIES Fortran_MODULE_DIRECTORY ${PROJECT_SOURCE_DIR}/WW3/model/mod) + set(WW3_LIBS ${PROJECT_SOURCE_DIR}/WW3/model/obj/libww3_multi_esmf.a) + add_dependencies(NEMS.exe ww3_nems) +endif() + +if(CCPP) + set(CCPP_LIBRARIES ccppdriver ccppphys ccpp) +else() + set(IPD_LIBRARIES ipd) +endif() + +target_link_libraries(NEMS.exe + fv3cap + fv3core + io + ${IPD_LIBRARIES} + ${PHYS}physics + ${CCPP_LIBRARIES} + fv3cpl + stochastic_physics + fms + ${WW3_LIBS} + ${NCEP_LIBS} ${ESMF_LIBS} ${NETCDF_LIBS} ${MKL_LIB}) + +############################################################################### +### done +############################################################################### diff --git a/FV3 b/FV3 index bc4aeabac6..ee9db83132 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit bc4aeabac69357a3403bb36fc3839586e11d9d41 +Subproject commit ee9db831320bec9583fbb61181c1132214a7bf03 diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..23ce9c67d0 --- /dev/null +++ b/build.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -eu + +MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) + +export COMPILER=${COMPILER:?"Please set COMPILER environment variable [gnu|intel]"} +export CMAKE_Platform=linux.${COMPILER} +export CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-mpicc} +export CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER:-mpicxx} +export CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-mpif90} + +export BACIO_LIB4=${BACIO_LIB4:?"Please set BACIO_LIB4 environment variable"} +export NEMSIO_INC=${NEMSIO_INC:?"Please set NEMSIO_INC environment variable"} +export NEMSIO_LIB=${NEMSIO_LIB:?"Please set NEMSIO_LIB environment variable"} +export SP_LIBd=${SP_LIBd:?"Please set SP_LIBd environment variable"} +export W3EMC_LIBd=${W3EMC_LIBd:?"Please set W3EMC_LIBd environment variable"} +export W3NCO_LIBd=${W3NCO_LIBd:?"Please set W3NCO_LIBd environment variable"} +export NETCDF=${NETCDF:?"Please set NETCDF environment variable"} +export ESMFMKFILE=${ESMFMKFILE:?"Please set ESMFMKFILE environment variable"} + +BUILD_DIR=${MYDIR}/build +rm -rf ${BUILD_DIR} +mkdir ${BUILD_DIR} + +CCPP_SUITES="${CCPP_SUITES:-FV3_GFS_2017_gfdlmp}" + +./FV3/ccpp/framework/scripts/ccpp_prebuild.py \ + --config=FV3/ccpp/config/ccpp_prebuild_config.py \ + --static \ + --suites=${CCPP_SUITES} \ + --builddir=${BUILD_DIR}/FV3 > ${BUILD_DIR}/ccpp_prebuild.log 2>&1 + +source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_SCHEMES.sh +source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_CAPS.sh +source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_STATIC_API.sh + +CMAKE_FLAGS+=" -DCCPP=ON -DSTATIC=ON -DSUITES=${CCPP_SUITES} -DNETCDF_DIR=${NETCDF}" + +cd ${BUILD_DIR} +cmake .. ${CMAKE_FLAGS} +make -j ${BUILD_JOBS:-4} +cp NEMS.exe ${MYDIR}/ufs_weather_model diff --git a/cmake/FindESMF.cmake b/cmake/FindESMF.cmake new file mode 100644 index 0000000000..175a394885 --- /dev/null +++ b/cmake/FindESMF.cmake @@ -0,0 +1,47 @@ + +if (DEFINED ENV{ESMFMKFILE}) + message("ESMFMKFILE: $ENV{ESMFMKFILE}") +else() + message(FATAL_ERROR "ESMFMKFILE env variable is not defined") +endif() + +set(ESMFMKFILE $ENV{ESMFMKFILE}) + +# convert esmf.mk makefile variables to cmake variables until ESMF +# provides proper cmake package +file(STRINGS ${ESMFMKFILE} esmf_mk_text) +foreach(line ${esmf_mk_text}) + string(REGEX REPLACE "^[ ]+" "" line ${line}) # strip leading spaces + if (line MATCHES "^ESMF_*") # process only line starting with ESMF_ + string(REGEX MATCH "^ESMF_[^=]+" esmf_name ${line}) + string(REPLACE "${esmf_name}=" "" emsf_value ${line}) + set(${esmf_name} "${emsf_value}") + endif() +endforeach() +string(REPLACE "-I" "" ESMF_F90COMPILEPATHS ${ESMF_F90COMPILEPATHS}) +string(REPLACE " " ";" ESMF_F90COMPILEPATHS ${ESMF_F90COMPILEPATHS}) + +# We use only these 4 variables in our build system. Make sure they are all set +if(ESMF_VERSION_MAJOR AND + ESMF_F90COMPILEPATHS AND + ESMF_F90ESMFLINKRPATHS AND + ESMF_F90ESMFLINKLIBS) + message(" Found ESMF:") + message("ESMF_VERSION_MAJOR: ${ESMF_VERSION_MAJOR}") + message("ESMF_F90COMPILEPATHS: ${ESMF_F90COMPILEPATHS}") + message("ESMF_F90ESMFLINKRPATHS: ${ESMF_F90ESMFLINKRPATHS}") + message("ESMF_F90ESMFLINKLIBS: ${ESMF_F90ESMFLINKLIBS}") +else() + message("One of the ESMF_ variables is not defined") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ESMF + FOUND_VAR + ESMF_FOUND + REQUIRED_VARS + ESMF_F90COMPILEPATHS + ESMF_F90ESMFLINKRPATHS + ESMF_F90ESMFLINKLIBS + VERSION_VAR + ESMF_VERSION_STRING) diff --git a/cmake/GNU.cmake b/cmake/GNU.cmake new file mode 100644 index 0000000000..a93b6f4ac8 --- /dev/null +++ b/cmake/GNU.cmake @@ -0,0 +1,63 @@ + +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fcray-pointer -ffree-line-length-none -fno-range-check -fbacktrace") + +if(DEBUG) + message("DEBUG is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Wall -O0 -ggdb -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O0 -ggdb") +else() + message("DEBUG is disabled (optimized build)") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -fno-range-check") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") +endif() + + +if(REPRO) + message("REPRO is ENABLED") + if(APPLE) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -ggdb") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -ggdb") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") + endif() +else() + message("REPRO is disabled") +endif() + + +if(32BIT) + message("32BIT is ENABLED") + add_definitions(-DOVERLOAD_R4) + add_definitions(-DOVERLOAD_R8) +else() + message("32BIT is disabled") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8") +endif() + + +if(OPENMP) + message("OPENMP is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenmp") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fopenmp") + add_definitions(-DOPENMP) +else() + message("OPENMP is disabled") +endif() + +if(AVX2) + message("AVX2 is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") +else() + message("AVX2 is disabled") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") +endif() + +if(INLINE_POST) + message("INLINE_POST is ENABLED") +else() + message("INLINE_POST is disabled") +endif() diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake new file mode 100644 index 0000000000..8fe97f2e5c --- /dev/null +++ b/cmake/Intel.cmake @@ -0,0 +1,92 @@ + +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte") +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qno-opt-dynamic-align") + + +if(32BIT) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -real-size 32") + add_definitions(-DOVERLOAD_R4) + add_definitions(-DOVERLOAD_R8) +else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -real-size 64") + if(NOT REPRO) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -no-prec-div -no-prec-sqrt") + endif() +endif() + +if(REPRO) + add_definitions(-DREPRO) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align") +else() + if(AVX2) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xHOST -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xHOST -qno-opt-dynamic-align") + endif() +endif() + +if(REPRO) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal") +else() + if(DEBUG) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -g -ftrapuv -traceback") + add_definitions(-DDEBUG) + else() + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal") + set(FAST "-fast-transcendentals") + endif() +endif() + +if(OPENMP) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qopenmp") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qopenmp") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -qopenmp") + add_definitions(-DOPENMP) +endif() + + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__IFC -sox -fp-model source") + +# print build options + +if(DEBUG) + message("DEBUG is ENABLED") +else() + message("DEBUG is disabled") +endif() + +if(REPRO) + message("REPRO is ENABLED") +else() + message("REPRO is disabled") +endif() + +if(32BIT) + message("32BIT is ENABLED") +else() + message("32BIT is disabled") +endif() + +if(OPENMP) + message("OPENMP is ENABLED") +else() + message("OPENMP is disabled") +endif() + +if(AVX2) + message("AVX2 is ENABLED") +else() + message("AVX2 is disabled") +endif() + +if(INLINE_POST) + message("INLINE_POST is ENABLED") +else() + message("INLINE_POST is disabled") +endif() diff --git a/cmake/PGI.cmake b/cmake/PGI.cmake new file mode 100644 index 0000000000..b5c01d91cf --- /dev/null +++ b/cmake/PGI.cmake @@ -0,0 +1,62 @@ + +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mpreprocess") + +if(DEBUG) + message("DEBUG is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0 -Ktrap=fp -Mbounds -traceback") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -g -traceback") +else() + message("DEBUG is disabled (optimized build)") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") + set(FAST "-fast-transcendentals") +endif() + + +if(REPRO) + message("REPRO is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O1 -g -traceback") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") +else() + message("REPRO is disabled") +endif() + + +if(32BIT) + message("32BIT is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -r4") + add_definitions(-DOVERLOAD_R4) + add_definitions(-DOVERLOAD_R8) +else() + message("32BIT is disabled") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -i4 -r8 -Mfprelaxed=div -Mfprelaxed=sqrt") +endif() + + +if(OPENMP) + message("OPENMP is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mp") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mp") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mp") + add_definitions(-DOPENMP) +else() + message("OPENMP is disabled") +endif() + + +if(AVX2) + message("AVX2 is ENABLED") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -tp=haswell") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -tp=haswell") +else() + message("AVX2 is disabled") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -tp=x64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -tp=x64") +endif() + + +if(INLINE_POST) + message("INLINE_POST is ENABLED") +else() + message("INLINE_POST is disabled") +endif() diff --git a/cmake/configure_gaea.cmake b/cmake/configure_gaea.cmake new file mode 100644 index 0000000000..e6dbc3cfe0 --- /dev/null +++ b/cmake/configure_gaea.cmake @@ -0,0 +1,32 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" ON) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(NCEP_LIBS $ENV{NEMSIO_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/cmake/configure_hera.intel.cmake b/cmake/configure_hera.intel.cmake new file mode 100644 index 0000000000..645406a868 --- /dev/null +++ b/cmake/configure_hera.intel.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" ON) + +option(INLINE_POST "Enable inline post" ON) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/cmake/configure_jet.cmake b/cmake/configure_jet.cmake new file mode 100644 index 0000000000..846fa0c7d3 --- /dev/null +++ b/cmake/configure_jet.cmake @@ -0,0 +1,42 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" OFF) + +option(INLINE_POST "Enable inline post" OFF) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +message("AVX2 is ENABLED on Jet (multi-tagret executable)") +string (REPLACE "-xHOST" "-axSSE4.2,AVX,CORE-AVX2" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") +string (REPLACE "-xHOST" "-axSSE4.2,AVX,CORE-AVX2" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + +string(REPLACE "-i_dynamic" "-shared-intel" + CMAKE_SHARED_LIBRARY_LINK_Fortran_FLAGS + "${CMAKE_SHARED_LIBRARY_LINK_Fortran_FLAGS}") + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(NCEP_LIBS $ENV{NEMSIO_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/cmake/configure_linux.gnu.cmake b/cmake/configure_linux.gnu.cmake new file mode 100644 index 0000000000..7efcb1633e --- /dev/null +++ b/cmake/configure_linux.gnu.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" OFF) + +option(INLINE_POST "Enable inline post" OFF) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/cmake/configure_linux.intel.cmake b/cmake/configure_linux.intel.cmake new file mode 100644 index 0000000000..7efcb1633e --- /dev/null +++ b/cmake/configure_linux.intel.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" OFF) + +option(INLINE_POST "Enable inline post" OFF) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/cmake/configure_macosx.gnu.cmake b/cmake/configure_macosx.gnu.cmake new file mode 100644 index 0000000000..027e2f8a57 --- /dev/null +++ b/cmake/configure_macosx.gnu.cmake @@ -0,0 +1,39 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" OFF) + +option(INLINE_POST "Enable inline post" OFF) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +set(MKL_DIR $ENV{MKL_DIR}) +set(MKL_INC $ENV{MKL_INC}) +set(MKL_LIB $ENV{MKL_LIB}) + +message("") diff --git a/cmake/configure_wcoss_cray.cmake b/cmake/configure_wcoss_cray.cmake new file mode 100644 index 0000000000..645406a868 --- /dev/null +++ b/cmake/configure_wcoss_cray.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" ON) + +option(INLINE_POST "Enable inline post" ON) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/cmake/configure_wcoss_dell_p3.cmake b/cmake/configure_wcoss_dell_p3.cmake new file mode 100644 index 0000000000..72dbf1dc9e --- /dev/null +++ b/cmake/configure_wcoss_dell_p3.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" OFF) + +option(INLINE_POST "Enable inline post" ON) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/modulefiles/gaea.intel/fv3 b/modulefiles/gaea.intel/fv3 index 9106f7562a..828deb2856 100644 --- a/modulefiles/gaea.intel/fv3 +++ b/modulefiles/gaea.intel/fv3 @@ -39,3 +39,12 @@ module load esmf/8.0.0 # Needed at runtime: module load alps + +## +## load cmake +## +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform gaea +setenv NETCDF ${NETCDF_DIR} diff --git a/modulefiles/gaea.intel/fv3.intel-16.0.3.210 b/modulefiles/gaea.intel/fv3.intel-16.0.3.210 index 407f07407e..67bb46ad0e 100644 --- a/modulefiles/gaea.intel/fv3.intel-16.0.3.210 +++ b/modulefiles/gaea.intel/fv3.intel-16.0.3.210 @@ -41,3 +41,14 @@ module load esmf/8.0.0_bs50 # Needed at runtime: module load alps + +setenv CC mpicc +setenv CXX mpicxx +setenv FC mpif90 +setenv F77 mpif77 +setenv F90 mpif90 + +setenv CMAKE_C_COMPILER mpicc +setenv CMAKE_CXX_COMPILER mpic++ +setenv CMAKE_Fortran_COMPILER mpif90 +setenv CMAKE_Platform theia.gnu diff --git a/modulefiles/hera.intel/fv3 b/modulefiles/hera.intel/fv3 index 761fad5b0d..a1d3cc721d 100644 --- a/modulefiles/hera.intel/fv3 +++ b/modulefiles/hera.intel/fv3 @@ -41,7 +41,6 @@ module load crtm/2.2.6 module load jasper/1.900.1 module load png/1.2.44 module load z/1.2.11 - ## load modules for nceppost grib module load post/8.0.0 @@ -51,3 +50,13 @@ module load post/8.0.0 ## module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles module load esmf/8.0.0 + +## +## load cmake +## +module use /contrib/modulefiles +module load cmake/3.9.0 +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform hera.intel diff --git a/modulefiles/jet.intel/fv3 b/modulefiles/jet.intel/fv3 index 9e8aab3806..9cdc8b20d6 100644 --- a/modulefiles/jet.intel/fv3 +++ b/modulefiles/jet.intel/fv3 @@ -31,3 +31,11 @@ module load NCEPlibs/9.9.9 ### use pre-compiled EMSF library for above compiler / MPI combination ## module load esmf/8.0.0 + +## +## load cmake +## +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform jet diff --git a/modulefiles/macosx.gnu/fv3 b/modulefiles/macosx.gnu/fv3 index c7d2a5c3a1..f145dfccbb 100755 --- a/modulefiles/macosx.gnu/fv3 +++ b/modulefiles/macosx.gnu/fv3 @@ -18,6 +18,14 @@ export CPP=${CPP:-"${F90} -E -x f95-cpp-input"} export MPICC=${MPICC:-mpicc} export MPIF90=${MPIF90:-mpif90} +## +## load cmake +## +export CMAKE_C_COMPILER=${CC} +export CMAKE_CXX_COMPILER=${CXX} +export CMAKE_Fortran_COMPILER=${FC} +export CMAKE_Platform=macosx.gnu + ## ## use homebrew NetCDF library ## diff --git a/modulefiles/wcoss_cray/fv3 b/modulefiles/wcoss_cray/fv3 index c5d5b5eca9..2aeb6ebb0e 100644 --- a/modulefiles/wcoss_cray/fv3 +++ b/modulefiles/wcoss_cray/fv3 @@ -61,3 +61,12 @@ module use /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles module load esmf/8.0.0 module swap pmi pmi/5.0.11 + +## +## load cmake +## +module load cmake/3.6.2 +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss_cray diff --git a/modulefiles/wcoss_dell_p3/fv3 b/modulefiles/wcoss_dell_p3/fv3 index 8386db8a51..477df7dc36 100644 --- a/modulefiles/wcoss_dell_p3/fv3 +++ b/modulefiles/wcoss_dell_p3/fv3 @@ -17,6 +17,7 @@ module-whatis "loads NEMS FV3 prerequisites on Venus and Mars" ## module load ips/18.0.1.163 module load impi/18.0.1 +module load lsf/10.1 module load NetCDF/4.5.0 module load HDF5-serial/1.10.1 module load cmake/3.10.0 @@ -45,3 +46,12 @@ module load post/8.0.0 module use /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/soft/modulefiles module load esmf/8.0.0 + +## +## load cmake +## +module load cmake/3.10.0 +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform wcoss_dell_p3 diff --git a/tests/compile_cmake.sh b/tests/compile_cmake.sh new file mode 100755 index 0000000000..b6c3855f8c --- /dev/null +++ b/tests/compile_cmake.sh @@ -0,0 +1,172 @@ +#!/bin/bash +set -eux + +function trim { + local var="$1" + # remove leading whitespace characters + var="${var#"${var%%[![:space:]]*}"}" + # remove trailing whitespace characters + var="${var%"${var##*[![:space:]]}"}" + echo -n "$var" +} + +SECONDS=0 + +readonly MYDIR=$( dirname $(readlink -f $0) ) + +# ---------------------------------------------------------------------- +# Parse arguments. + +readonly ARGC=$# + +cd ${MYDIR} + +if [[ $ARGC -eq 0 ]]; then + COMPILER=intel + . detect_machine.sh + PATHTR=$(cd -P ${MYDIR}/.. && pwd) + MAKE_OPT='' + BUILD_NAME=fv3 + clean_before=YES + clean_after=YES +elif [[ $ARGC -lt 2 ]]; then + echo "Usage: $0 PATHTR MACHINE_ID [ MAKE_OPT [ BUILD_NR ] [ clean_before ] [ clean_after ] ]" + echo Valid MACHINE_IDs: + echo $( ls -1 ../conf/configure.fv3.* | sed s,.*fv3\.,,g ) | fold -sw72 + exit 1 +else + PATHTR=$1 + MACHINE_ID=$2 + MAKE_OPT=${3:-} + BUILD_NAME=fv3${4:+_$4} + clean_before=${5:-YES} + clean_after=${6:-YES} +fi +BUILD_DIR=build_${BUILD_NAME} + +# ---------------------------------------------------------------------- +# Make sure we have reasonable number of threads. + +if [[ $MACHINE_ID == cheyenne.* ]] ; then + MAKE_THREADS=${MAKE_THREADS:-3} +elif [[ $MACHINE_ID == wcoss_dell_p3 ]] ; then + MAKE_THREADS=${MAKE_THREADS:-4} +fi + +MAKE_THREADS=${MAKE_THREADS:-8} + +hostname + +cd ${PATHTR}/tests + +# ---------------------------------------------------------------------- + +echo "Compiling ${MAKE_OPT} into $BUILD_NAME.exe on $MACHINE_ID" + +if [ $clean_before = YES ] ; then + rm -rf ${BUILD_DIR} +fi + +mkdir -p ${BUILD_DIR} + +# set CCPP_CMAKE_FLAGS based on $MAKE_OPT + +CCPP_CMAKE_FLAGS="" + +if [[ "${MAKE_OPT}" == *"DEBUG=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DDEBUG=Y" +elif [[ "${MAKE_OPT}" == *"REPRO=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DREPRO=Y" +fi + +if [[ "${MAKE_OPT}" == *"32BIT=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -D32BIT=Y" +fi + +if [[ "${MAKE_OPT}" == *"OPENMP=N"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DOPENMP=OFF" +fi + +if [[ "${MAKE_OPT}" == *"CCPP=Y"* ]]; then + + # Account for inconsistencies in HPC modules: if environment variable + # NETCDF is undefined, try to set from NETCDF_DIR, NETCDF_ROOT, ... + if [[ "${MACHINE_ID}" == "wcoss_cray" ]]; then + NETCDF=${NETCDF:-${NETCDF_DIR}} + fi + + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DNETCDF_DIR=${NETCDF} -DCCPP=ON -DMPI=ON" + + if [[ "${MAKE_OPT}" == *"DEBUG=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DCMAKE_BUILD_TYPE=Debug" + elif [[ "${MAKE_OPT}" == *"REPRO=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DCMAKE_BUILD_TYPE=Bitforbit" + else + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DCMAKE_BUILD_TYPE=Release" + if [[ "${MACHINE_ID}" == "jet.intel" ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DSIMDMULTIARCH=ON" + fi + fi + + if [[ "${MAKE_OPT}" == *"32BIT=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DDYN32=ON" + else + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DDYN32=OFF" + fi + + if [[ "${MAKE_OPT}" == *"STATIC=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DSTATIC=ON" + else + echo "Error, cmake build not compatible with dynamic CCPP" + exit 1 + fi + + if [[ "${MAKE_OPT}" == *"MULTI_GASES=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DMULTI_GASES=ON" + else + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DMULTI_GASES=OFF" + fi + + ( + SUITES=$( echo $MAKE_OPT | sed 's/.* SUITES=//' | sed 's/ .*//' ) + cd ${PATHTR} + ./FV3/ccpp/framework/scripts/ccpp_prebuild.py --config=FV3/ccpp/config/ccpp_prebuild_config.py --static --suites=${SUITES} --builddir=tests/${BUILD_DIR}/FV3 + ) + + # Read list of schemes, caps, and static API + source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_SCHEMES.sh + source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_CAPS.sh + source ${BUILD_DIR}/FV3/ccpp/physics/CCPP_STATIC_API.sh + + fi + +if [[ "${MAKE_OPT}" == *"NAM_phys=Y"* ]]; then + CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DPHYS=nam" +fi +CCPP_CMAKE_FLAGS=$(trim "${CCPP_CMAKE_FLAGS}") + +( + source $PATHTR/NEMS/src/conf/module-setup.sh.inc + if [[ $MACHINE_ID == macosx.* ]] || [[ $MACHINE_ID == linux.* ]]; then + source $PATHTR/modulefiles/${MACHINE_ID}/fv3 + else + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + fi + + cd ${BUILD_DIR} + + cmake ${PATHTR} ${CCPP_CMAKE_FLAGS} + make -j ${MAKE_THREADS} + mv NEMS.exe ../${BUILD_NAME}.exe + cp ${PATHTR}/modulefiles/${MACHINE_ID}/fv3 ../modules.${BUILD_NAME} + cd .. +) + +if [ $clean_after = YES ] ; then + rm -rf ${BUILD_DIR} +fi + +elapsed=$SECONDS +echo "Elapsed time $elapsed seconds. Compiling ${MAKE_OPT} finished" + diff --git a/tests/fv3_conf/fv3_slurm.IN_gaea b/tests/fv3_conf/fv3_slurm.IN_gaea old mode 100755 new mode 100644 diff --git a/tests/fv3_conf/fv3_slurm.IN_hera b/tests/fv3_conf/fv3_slurm.IN_hera old mode 100755 new mode 100644 diff --git a/tests/fv3_conf/fv3_slurm.IN_theia b/tests/fv3_conf/fv3_slurm.IN_theia old mode 100755 new mode 100644 diff --git a/tests/rt.conf b/tests/rt.conf index 1d31b6b393..eec0894999 100644 --- a/tests/rt.conf +++ b/tests/rt.conf @@ -64,7 +64,7 @@ RUN | fv3_stretched_nest_quilt RUN | fv3_stretched_nest_quilt | standard | cheyenne.intel | fv3 | RUN | fv3_regional_control | standard | | fv3 | -RUN | fv3_regional_restart | standard | | fv3 | +RUN | fv3_regional_restart | standard | | fv3 | fv3_regional_control RUN | fv3_regional_quilt | standard | | fv3 | # fv3_regional_c768 not working on Cheyenne, code aborts with invalid values # for surface pressure, out of range warnings and all other sorts of errors @@ -137,7 +137,7 @@ RUN | fv3_gfdlmprad_32bit_post RUN | fv3_stretched | standard | | fv3 | RUN | fv3_stretched_nest | standard | | fv3 | RUN | fv3_regional_control | standard | | fv3 | -RUN | fv3_regional_restart | standard | | fv3 | +RUN | fv3_regional_restart | standard | | fv3 | fv3_regional_control RUN | fv3_regional_quilt | standard | | fv3 | # fv3_regional_c768 not working on Cheyenne, code aborts with invalid values # for surface pressure, out of range warnings and all other sorts of errors @@ -205,7 +205,7 @@ COMPILE | REPRO=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_20 COMPILE | REPRO=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y | standard | jet.intel | | RUN | fv3_ccpp_regional_control | standard | | | -RUN | fv3_ccpp_regional_restart | standard | | | +RUN | fv3_ccpp_regional_restart | standard | | | fv3_ccpp_regional_control RUN | fv3_ccpp_regional_quilt | standard | | | # fv3_regional_c768 not working on Cheyenne, code aborts with invalid values # for surface pressure, out of range warnings and all other sorts of errors @@ -308,7 +308,7 @@ COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlm COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y | standard | jet.intel | fv3 | RUN | fv3_ccpp_regional_control | standard | | fv3 | -RUN | fv3_ccpp_regional_restart | standard | | fv3 | +RUN | fv3_ccpp_regional_restart | standard | | fv3 | fv3_ccpp_regional_control RUN | fv3_ccpp_regional_quilt | standard | | fv3 | # fv3_regional_c768 not working on Cheyenne, code aborts with invalid values # for surface pressure, out of range warnings and all other sorts of errors diff --git a/tests/rt.sh b/tests/rt.sh index 5ec70ae076..8be1e7ddeb 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -1,6 +1,8 @@ #!/bin/bash set -eux +SECONDS=0 + hostname die() { echo "$@" >&2; exit 1; } @@ -27,11 +29,11 @@ usage() { rt_trap() { [[ ${ROCOTO:-false} == true ]] && rocoto_kill - [[ ${ECFLOW:-false} == true ]] && { ecflow_kill; ecflow_stop; } cleanup } cleanup() { + [[ ${ECFLOW:-false} == true ]] && ecflow_stop rm -rf ${LOCKDIR} trap 0 exit @@ -88,14 +90,22 @@ elif [[ $MACHINE_ID = wcoss_cray ]]; then source $PATHTR/NEMS/src/conf/module-setup.sh.inc module load xt-lsfhpc + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + module load python/2.7.14 + module use /usrx/local/emc_rocoto/modulefiles - module load rocoto/1.2.4-RC3 + module load rocoto/1.3.0rc2 ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + module load ecflow/intel/4.7.1 ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') + DISKNM=/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT QUEUE=debug PARTITION= @@ -114,15 +124,23 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then source $PATHTR/NEMS/src/conf/module-setup.sh.inc module load lsf/10.1 + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + module load python/2.7.14 + module use /usrx/local/dev/emc_rocoto/modulefiles - module load ruby/2.5.1 rocoto/complete + module load ruby/2.5.1 rocoto/1.3.0rc2 ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + module load ips/18.0.1.163 module load ecflow/4.7.1 ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') + DISKNM=/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT QUEUE=debug PARTITION= @@ -159,15 +177,20 @@ elif [[ $MACHINE_ID = hera.* ]]; then export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib source $PATHTR/NEMS/src/conf/module-setup.sh.inc + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + # Re-instantiate COMPILER in case it gets deleted by module purge COMPILER=${NEMS_COMPILER:-intel} module load rocoto ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) - export PATH=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/bin:$PATH - export PYTHONPATH=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/lib/python2.7/site-packages - ECFLOW_START=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/bin/ecflow_start.sh + ROCOTOCOMPLETE=$(which rocotocomplete) + export PATH=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ecflow/bin:$PATH + export PYTHONPATH=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ecflow/lib/python2.7/site-packages + ECFLOW_START=/scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/ecflow/bin/ecflow_start.sh ECF_PORT=$(( $(id -u) + 1500 )) QUEUE=debug # ACCNR=fv3-cpu @@ -177,49 +200,59 @@ elif [[ $MACHINE_ID = hera.* ]]; then STMP=$dprefix/stmp4 PTMP=$dprefix/stmp2 - # default scheduler on Hera SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_hera fv3_conf/fv3_slurm.IN elif [[ $MACHINE_ID = theia.* ]]; then source $PATHTR/NEMS/src/conf/module-setup.sh.inc + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + # Re-instantiate COMPILER in case it gets deleted by module purge COMPILER=${NEMS_COMPILER:-intel} - module load rocoto + module load rocoto/1.3.0 ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) export PATH=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/bin:$PATH export PYTHONPATH=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/lib/python2.7/site-packages ECFLOW_START=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/bin/ecflow_start.sh ECF_PORT=$(( $(id -u) + 1500 )) QUEUE=debug -# ACCNR=fv3-cpu +# ACCNR= # detected in detect_machine.sh PARTITION= dprefix=/scratch4/NCEPDEV DISKNM=$dprefix/nems/noscrub/emc.nemspara/RT STMP=$dprefix/stmp4 PTMP=$dprefix/stmp3 - # default scheduler on Theia SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_theia fv3_conf/fv3_slurm.IN elif [[ $MACHINE_ID = jet.* ]]; then source $PATHTR/NEMS/src/conf/module-setup.sh.inc + + module use $PATHTR/modulefiles/${MACHINE_ID} + module load fv3 + # Re-instantiate COMPILER in case it gets deleted by module purge COMPILER=${NEMS_COMPILER:-intel} - module load rocoto + module load rocoto/1.3.1 ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) - export PATH=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/bin:$PATH - export PYTHONPATH=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/lib/python2.6/site-packages - ECFLOW_START=/scratch4/NCEPDEV/meso/save/Dusan.Jovic/ecflow/bin/ecflow_start.sh + ROCOTOCOMPLETE=$(which rocotocomplete) + + export PATH=/mnt/lfs3/projects/hfv3gfs/Dusan.Jovic/ecflow/bin:$PATH + export PYTHONPATH=/mnt/lfs3/projects/hfv3gfs/Dusan.Jovic/ecflow/lib/python2.7/site-packages + ECFLOW_START=/mnt/lfs3/projects/hfv3gfs/Dusan.Jovic/ecflow/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) QUEUE=debug -# ACCNR=fv3-cpu + ACCNR=hfv3gfs PARTITION=xjet DISKNM=/lfs3/projects/hfv3gfs/GMTB/RT dprefix=/lfs3/projects/hfv3gfs/$USER @@ -405,7 +438,7 @@ if [[ $ROCOTO == true ]]; then ROCOTO_XML=${PATHRT}/rocoto_workflow.xml ROCOTO_DB=${PATHRT}/rocoto_workflow.db - rm -f $ROCOTO_XML $ROCOTO_DB + rm -f $ROCOTO_XML $ROCOTO_DB *_lock.db if [[ $MACHINE_ID = wcoss ]]; then QUEUE=dev @@ -421,16 +454,16 @@ if [[ $ROCOTO == true ]]; then ROCOTO_SCHEDULER=lsf elif [[ $MACHINE_ID = hera.* ]]; then QUEUE=batch - COMPILE_QUEUE=service - ROCOTO_SCHEDULER=moabtorque + COMPILE_QUEUE=batch + ROCOTO_SCHEDULER=slurm elif [[ $MACHINE_ID = theia.* ]]; then QUEUE=batch - COMPILE_QUEUE=service - ROCOTO_SCHEDULER=moabtorque + COMPILE_QUEUE=batch + ROCOTO_SCHEDULER=slurm elif [[ $MACHINE_ID = jet.* ]]; then QUEUE=batch - COMPILE_QUEUE=service - ROCOTO_SCHEDULER=moabtorque + COMPILE_QUEUE=batch + ROCOTO_SCHEDULER=slurm else die "Rocoto is not supported on this machine $MACHINE_ID" fi @@ -466,7 +499,9 @@ suite regtest edit ECF_INCLUDE '${ECFLOW_RUN}' edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1 edit ECF_TRIES 1 - limit max_jobs 10 + label rundir_root '${RUNDIR_ROOT}' + limit max_builds 6 + limit max_jobs 30 EOF if [[ $MACHINE_ID = wcoss ]]; then @@ -495,6 +530,8 @@ fi new_compile=false in_metatask=false +[[ -f $TESTS_FILE ]] || die "$TESTS_FILE does not exist" + while read -r line; do line="${line#"${line%%[![:space:]]*}"}" @@ -513,7 +550,6 @@ while read -r line; do [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue - COMPILE_NR_DEP=${COMPILE_NR} (( COMPILE_NR += 1 )) if [[ $ROCOTO == true ]]; then @@ -521,8 +557,8 @@ while read -r line; do elif [[ $ECFLOW == true ]]; then ecflow_create_compile_task else - ./compile.sh $PATHTR/FV3 $MACHINE_ID "${NEMS_VER}" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 -# ./compile.sh $PATHTR/FV3 $MACHINE_ID DEBUG=Y $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 + #./compile.sh $PATHTR/FV3 $MACHINE_ID "${NEMS_VER}" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 + ./compile_cmake.sh $PATHTR $MACHINE_ID "${NEMS_VER}" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 echo " bash Compile is done" fi @@ -540,7 +576,7 @@ while read -r line; do elif [[ $line == APPBUILD* ]] ; then - APP=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') + APP=$( echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') SET=$( echo $line | cut -d'|' -f3) MACHINES=$(echo $line | cut -d'|' -f4 | sed -e 's/^ *//' -e 's/ *$//') CB=$( echo $line | cut -d'|' -f5) @@ -548,8 +584,8 @@ while read -r line; do [[ $SET_ID != ' ' && $SET != *${SET_ID}* ]] && continue [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue + [[ ${ROCOTO} == true || ${ECFLOW} == true ]] && continue - COMPILE_NR_DEP=${COMPILE_NR} (( COMPILE_NR += 1 )) if [[ $ROCOTO == true ]]; then @@ -589,24 +625,38 @@ while read -r line; do [[ $MACHINES != ' ' && $MACHINES != *${MACHINE_ID}* ]] && continue [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue + # skip all *_appbuild runs if rocoto or ecFlow is used. FIXME + if [[ ${ROCOTO} == true && ${ECFLOW} == true ]]; then + if [[ ${TEST_NAME} == *_appbuild ]]; then + continue + fi + fi + # Avoid uninitialized RT_SUFFIX/BL_SUFFIX (see definition above) RT_SUFFIX=${RT_SUFFIX:-""} BL_SUFFIX=${BL_SUFFIX:-""} + if [[ $MACHINE_ID = wcoss_cray ]]; then + if [[ $RT_SUFFIX != "" || $BL_SUFFIX != "" ]]; then + # skip all REPRO and/or CCPP runs on wcoss_cray. FIXME + continue + fi + fi + if [[ $ROCOTO == true && $new_compile == true ]]; then new_compile=false in_metatask=true cat << EOF >> $ROCOTO_XML - 0 + 0 EOF fi - TEST_NR=$( printf '%02d' $(( 10#$TEST_NR + 1 )) ) + TEST_NR=$( printf '%03d' $(( 10#$TEST_NR + 1 )) ) ( source ${PATHRT}/tests/$TEST_NAME - cat << EOF > run_test.env + cat << EOF > ${RUNDIR_ROOT}/run_test_${TEST_NR}.env export MACHINE_ID=${MACHINE_ID} export RTPWD=${RTPWD} export PATHRT=${PATHRT} @@ -655,7 +705,6 @@ if [[ $ECFLOW == true ]]; then echo "endsuite" >> ${ECFLOW_RUN}/regtest.def # run ecflow workflow until done ecflow_run - ecflow_stop fi ## @@ -678,12 +727,13 @@ else echo ; echo REGRESSION TEST WAS SUCCESSFUL (echo ; echo REGRESSION TEST WAS SUCCESSFUL) >> ${REGRESSIONTEST_LOG} - rm -f fv3_*.x fv3_*.exe modules.fv3_* run_test.env + rm -f fv3_*.x fv3_*.exe modules.fv3_* [[ ${KEEP_RUNDIR} == false ]] && rm -rf ${RUNDIR_ROOT} - [[ ${ROCOTO:-false} == true ]] && rm -f ${ROCOTO_XML} ${ROCOTO_DB} - [[ ${ECFLOW:-false} == true ]] && rm -rf ${ECFLOW_RUN} + [[ ${ROCOTO} == true ]] && rm -f ${ROCOTO_XML} ${ROCOTO_DB} *_lock.db fi date >> ${REGRESSIONTEST_LOG} -exit +elapsed_time=$( printf '%02dh:%02dm:%02ds\n' $(($SECONDS%86400/3600)) $(($SECONDS%3600/60)) $(($SECONDS%60)) ) +echo "Elapsed time: ${elapsed_time}. Have a nice day!" >> ${REGRESSIONTEST_LOG} +echo "Elapsed time: ${elapsed_time}. Have a nice day!" diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index 8a7efc14cd..8e2246fca6 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -221,7 +221,7 @@ submit_and_wait() { state=$( sacct -n -j ${slurm_id}.batch --format=JobID,state,Jobname | grep ${slurm_id} | awk '{print $2}' ) echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is ${state}" fi - + elif [[ $SCHEDULER = 'sbatch' ]]; then #status=$( qstat -u ${USER} -n | grep ${JBNME} | awk '{print $"10"}' ); status=${status:--} PJP comment out to speed up regression test @@ -315,7 +315,7 @@ check_results() { # Give one minute for data to show up on file system #sleep 60 - echo >> ${REGRESSIONTEST_LOG} + echo > ${REGRESSIONTEST_LOG} echo "baseline dir = ${RTPWD}/${CNTL_DIR}" >> ${REGRESSIONTEST_LOG} echo "working dir = ${RUNDIR}" >> ${REGRESSIONTEST_LOG} echo "Checking test ${TEST_NR} ${TEST_NAME} results ...." >> ${REGRESSIONTEST_LOG} @@ -425,6 +425,8 @@ kill_job() { : elif [[ $SCHEDULER = 'pbs' ]]; then qdel ${jobid} + elif [[ $SCHEDULER = 'slurm' ]]; then + scancel ${jobid} elif [[ $SCHEDULER = 'lsf' ]]; then bkill ${jobid} fi @@ -442,49 +444,45 @@ rocoto_create_compile_task() { if [[ "Q$APP" != Q ]] ; then rocoto_cmd="&PATHRT;/appbuild.sh &PATHTR;/FV3 $APP $COMPILE_NR" else - rocoto_cmd="&PATHRT;/compile.sh &PATHTR;/FV3 $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR" + rocoto_cmd="&PATHRT;/compile_cmake.sh &PATHTR; $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR" fi NATIVE="" + BUILD_CORES=8 if [[ ${MACHINE_ID} == wcoss_dell_p3 ]]; then - NATIVE="6G -R 'affinity[core(1)]'" + BUILD_CORES=1 + NATIVE="8G -R 'affinity[core(1)]'" + fi + if [[ ${MACHINE_ID} == wcoss_cray ]]; then + BUILD_CORES=24 + rocoto_cmd="aprun -n 1 -j 1 -N 1 -d $BUILD_CORES $rocoto_cmd" + NATIVE="" + fi + BUILD_WALLTIME="00:30:00" + if [[ ${MACHINE_ID} == jet ]]; then + BUILD_WALLTIME="01:00:00" fi - if [[ ${COMPILE_NR_DEP} -gt 0 ]]; then - cat << EOF >> $ROCOTO_XML - - - $rocoto_cmd - compile_${COMPILE_NR} - ${ACCNR} - ${COMPILE_QUEUE} - 1 - 01:00:00 - &LOG;/compile_${COMPILE_NR}.log - ${NATIVE} - -EOF - else - cat << EOF >> $ROCOTO_XML - + cat << EOF >> $ROCOTO_XML + $rocoto_cmd compile_${COMPILE_NR} ${ACCNR} ${COMPILE_QUEUE} - 1 - 01:00:00 + ${PARTITION} + ${BUILD_CORES} + ${BUILD_WALLTIME} &LOG;/compile_${COMPILE_NR}.log ${NATIVE} EOF - fi } rocoto_create_run_task() { if [[ $DEP_RUN != '' ]]; then - DEP_STRING=" " + DEP_STRING=" " else DEP_STRING="" fi @@ -506,15 +504,16 @@ rocoto_create_run_task() { fi cat << EOF >> $ROCOTO_XML - + $DEP_STRING &PATHRT;/run_test.sh &PATHRT; &RUNDIR_ROOT; ${TEST_NAME} ${TEST_NR} ${COMPILE_NR} - ${TEST_NAME} + ${TEST_NAME}${RT_SUFFIX} ${ACCNR} ${QUEUE} + ${PARTITION} ${CORES} 00:${WLCLK}:00 - &LOG;/run_${TEST_NAME}.log + &LOG;/run_${TEST_NR}_${TEST_NAME}${RT_SUFFIX}.log ${NATIVE} EOF @@ -533,9 +532,14 @@ rocoto_run() { state="Active" while [[ $state != "Done" ]] do - $ROCOTORUN -w $ROCOTO_XML -d $ROCOTO_DB + $ROCOTORUN -v 10 -w $ROCOTO_XML -d $ROCOTO_DB sleep 10 & wait $! state=$($ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB -s | grep 197001010000 | awk -F" " '{print $2}') + dead_compile=$($ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB | grep compile_ | grep DEAD | head -1 | awk -F" " '{print $2}') + if [[ ! -z ${dead_compile} ]]; then + echo "y" | ${ROCOTOCOMPLETE} -w $ROCOTO_XML -d $ROCOTO_DB -m ${dead_compile}_tasks + ${ROCOTOCOMPLETE} -w $ROCOTO_XML -d $ROCOTO_DB -t ${dead_compile} + fi sleep 20 & wait $! done @@ -548,7 +552,7 @@ ecflow_create_compile_task() { if [[ "Q$APP" != Q ]] ; then ecflow_cmd="$PATHRT/appbuild.sh ${PATHTR}/FV3 $APP $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1" else - ecflow_cmd="$PATHRT/compile.sh ${PATHTR}/FV3 $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1" + ecflow_cmd="$PATHRT/compile_cmake.sh ${PATHTR} $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1" fi cat << EOF > ${ECFLOW_RUN}/regtest/compile_${COMPILE_NR}.ecf @@ -558,24 +562,21 @@ $ecflow_cmd EOF echo " task compile_${COMPILE_NR}" >> ${ECFLOW_RUN}/regtest.def - if [[ ${COMPILE_NR_DEP} -gt 0 ]]; then - echo " trigger compile_${COMPILE_NR_DEP} == complete" >> ${ECFLOW_RUN}/regtest.def - fi - + echo " inlimit max_builds" >> ${ECFLOW_RUN}/regtest.def } ecflow_create_run_task() { - cat << EOF > ${ECFLOW_RUN}/regtest/${TEST_NAME}.ecf + cat << EOF > ${ECFLOW_RUN}/regtest/${TEST_NAME}${RT_SUFFIX}.ecf %include -$PATHRT/run_test.sh ${PATHRT} ${RUNDIR_ROOT} ${TEST_NAME} ${TEST_NR} ${COMPILE_NR} > ${LOG_DIR}/run_${TEST_NAME}.log 2>&1 & +$PATHRT/run_test.sh ${PATHRT} ${RUNDIR_ROOT} ${TEST_NAME} ${TEST_NR} ${COMPILE_NR} > ${LOG_DIR}/run_${TEST_NR}_${TEST_NAME}${RT_SUFFIX}.log 2>&1 & %include EOF - echo " task ${TEST_NAME}" >> ${ECFLOW_RUN}/regtest.def + echo " task ${TEST_NAME}${RT_SUFFIX}" >> ${ECFLOW_RUN}/regtest.def echo " inlimit max_jobs" >> ${ECFLOW_RUN}/regtest.def if [[ $DEP_RUN != '' ]]; then - echo " trigger compile_${COMPILE_NR} == complete and ${DEP_RUN} == complete" >> ${ECFLOW_RUN}/regtest.def + echo " trigger compile_${COMPILE_NR} == complete and ${DEP_RUN}${RT_SUFFIX} == complete" >> ${ECFLOW_RUN}/regtest.def else echo " trigger compile_${COMPILE_NR} == complete" >> ${ECFLOW_RUN}/regtest.def fi @@ -584,31 +585,33 @@ EOF ecflow_run() { # in rare instances when UID is greater then 58500 (like Ratko's UID on theia) - [[ $ECF_PORT -ge 60000 ]] && ECF_PORT=12179 + [[ $ECF_PORT -gt 49151 ]] && ECF_PORT=12179 ECF_HOST=$( hostname ) - set +e - i=0 - max_atempts=5 - while [[ $i -lt $max_atempts ]]; do - ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} - not_running=$? - if [[ $not_running -eq 1 ]]; then - echo "ecflow_server is NOT running on ${ECF_HOST}:${ECF_PORT}" - sh ${ECFLOW_START} -d ${ECFLOW_RUN} -p ${ECF_PORT} >> ${ECFLOW_RUN}/ecflow.log 2>&1 - break - else - echo "ecflow_server is already running on ${ECF_HOST}:${ECF_PORT}" - ECF_PORT=$(( ECF_PORT + 1 )) - fi - i=$(( i + 1 )) - if [[ $i -eq $max_atempts ]]; then - echo "You already have $max_atempts ecFlow servers running on this node" - exit 1 - fi - done - set -e + sh ${ECFLOW_START} -d ${ECFLOW_RUN} -p ${ECF_PORT} >> ${ECFLOW_RUN}/ecflow.log 2>&1 + +# set +e +# i=0 +# max_atempts=5 +# while [[ $i -lt $max_atempts ]]; do +# ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} +# not_running=$? +# if [[ $not_running -eq 1 ]]; then +# echo "ecflow_server is NOT running on ${ECF_HOST}:${ECF_PORT}" +# sh ${ECFLOW_START} -d ${ECFLOW_RUN} -p ${ECF_PORT} >> ${ECFLOW_RUN}/ecflow.log 2>&1 +# break +# else +# echo "ecflow_server is already running on ${ECF_HOST}:${ECF_PORT}" +# ECF_PORT=$(( ECF_PORT + 1 )) +# fi +# i=$(( i + 1 )) +# if [[ $i -eq $max_atempts ]]; then +# echo "You already have $max_atempts ecFlow servers running on this node" +# exit 1 +# fi +# done +# set -e ECFLOW_RUNNING=true @@ -625,6 +628,7 @@ ecflow_run() { active_tasks=$( ecflow_client --get_state /regtest | grep "task " | grep -E 'state:active|state:submitted|state:queued' | wc -l ) ${PATHRT}/abort_dep_tasks.py done + sleep 65 # wait one ECF_INTERVAL plus 5 seconds } ecflow_kill() { diff --git a/tests/rt_wcoss_cray.conf b/tests/rt_wcoss_cray.conf index 4775eb987e..c8076cd466 100644 --- a/tests/rt_wcoss_cray.conf +++ b/tests/rt_wcoss_cray.conf @@ -54,7 +54,7 @@ RUN | fv3_stretched_nest_quilt RUN | fv3_stretched_nest_quilt | standard | cheyenne.intel | fv3 | RUN | fv3_regional_control | standard | | fv3 | -RUN | fv3_regional_restart | standard | | fv3 | +RUN | fv3_regional_restart | standard | | fv3 | fv3_regional_control RUN | fv3_regional_quilt | standard | | fv3 | # fv3_regional_c768 not working on Cheyenne, code aborts with invalid values # for surface pressure, out of range warnings and all other sorts of errors; this diff --git a/tests/run_test.sh b/tests/run_test.sh index 6a5441f915..67e5e041c4 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -16,7 +16,7 @@ export COMPILE_NR=$5 cd ${PATHRT} -[[ -e run_test.env ]] && source run_test.env +[[ -e ${RUNDIR_ROOT}/run_test_${TEST_NR}.env ]] && source ${RUNDIR_ROOT}/run_test_${TEST_NR}.env source default_vars.sh source tests/$TEST_NAME