Initial upload of pot3d package. More work on README needed, as well …

…as proper documentation of namelist options. Also, examples are pretty much empty and need to be developed.

README.md

@@ -1,3 +1,87 @@
 ![](pot3d_logo.png)
 
 POT3D: High Performance Potential Field Solver
+Predictive Science Inc.
+www.predsci.com
+
+###############################
+ HOW TO BUILD:
+###############################
+
+Modify "build.sh" to set the proper HDF5 library paths/flags 
+and compiler flags for your system.
+Then, simply run "./build.sh". 
+See comments inside build.sh for more details.
+
+###################################
+ HOW TO RUN SMALL TEST VALIDATION:
+###################################
+
+After building the code, simply run "./validate.sh".
+This will perform 2 runs of a small case using 1 and 2 MPI ranks respectively.
+
+The runs are performed in "testsuite/validation/run/" and the second run overwrites the first.
+
+Each result will be checked against a reference solution (in /runs/validation/validation) and
+a PASS/FAIL message will be displayed.
+
+
+
+###############################
+ POT3D DOCUMENTATION
+###############################
+
+POT3D uses a namelist in an input file called "pot3d.dat" to set all parameters of a run.
+It also requires an input 2D data set in HDF5 format to use for the lower boundary condition.
+This file in the tests provided are called 'br\_input\_<TEST>.h5' where <TEST> is the specific test name.
+
+### RUNNING POT3D ###
+
+Simply run the code as:
+ > mpiexec -np <NtotalRanks> ./pot3d
+(or ibrun, mpirun, etc).
+
+For CPU runs, please ensure "ifprec=2" is set in pot3d.dat 
+in order to ensure optimal computation speed.
+
+### RUNNING ON GPUs ###
+
+For standard cases, one should launch the code such that
+the number of MPI ranks per node is equal to the number of GPUs per node
+e.g.
+> mpiexec -np <NtotalRanks> --ntasks-per-node 4 ./pot3d
+or
+> mpiexec -np <NtotalRanks> --npersocket 2 ./pot3d
+etc.
+
+To run efficiently, it is critical that "_ifprec=1_" is set in pot3d.dat.
+
+### RUN OUTPUT ###
+
+At the end of the run, POT3D (as setup in the tests) outputs the following files:
+
+ pot3d.out An output log showing grid information and derived magnetic energy.
+ timing.out Results of the MPI timing recorded in the code.
+
+
+### TO VALIDATE ###
+
+We have included a validation script (pot3d\_validate.sh) that takes two pot3d.out files and
+compares the magnetic energy values in them. If they match, the test passes,
+if they do not, their values are displayed for further investigation.
+
+Example:
+./pot3d\_validate.sh ./runs/test/run/pot3d.out ./runs/test/validation/pot3d.out
+
+Within each test directory, there is a "validation" folder
+with the pot3d.out file for that run in it.
+
+Information on the computer, compiler, and setup used to generate the
+validation data is in the "validation\_run\_information.txt" file in
+each validation folder.
+
+
+### MEMORY FOOTPRINT ###
+To estimate how many GB of RAM is needed for a run,
+compute: nr*nt*np*8*15/1024/1000/1000
+

build.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+###################################################################
+# This build assumes that you have an "mpif90" in your PATH that is
+# set up to use your chosen MPI library and compiler.
+##################################################################
+
+#################################################################
+# Please set the location of your HDF5 include and library files. 
+# Make sure the HDF5 library is compiled with 
+# the same compiler currently being used and that the 
+# library is in your run-time environment (e.g. LD_LIBRARY_PATH).
+#################################################################
+
+#HDF5_INCLUDE_DIR="/usr/include/hdf5/serial"
+#HDF5_LIB_DIR="/usr/lib/x86_64-linux-gnu"
+
+HDF5_INCLUDE_DIR="/opt/psi/nv/ext_deps/deps/hdf5/include"
+HDF5_LIB_DIR="/opt/psi/nv/ext_deps/deps/hdf5/lib"
+
+###########################################################################
+# Please set the HDF5 linker flags to match your installed version of hdf5.
+###########################################################################
+
+#HDF5_LIB_FLAGS="-lhdf5_serial_fortran -lhdf5_serialhl_fortran -lhdf5_serial -lhdf5_serial_hl"
+HDF5_LIB_FLAGS="-lhdf5_fortran -lhdf5hl_fortran -lhdf5 -lhdf5_hl"
+
+###########################################################################
+# Please set the compile flags based on your compiler and hardware setup.
+# Examples:
+# GNU (CPU): FFLAGS="-O3 -mtune=native "
+# NV/PGI (CPU): FFLAGS="-O3"
+# NV/PGI (GPU): FFLAGS="-O3 -acc=gpu -gpu=cc60,cc70,cuda11.1 -Minfo=accel"
+# INTEL (CPU): FFLAGS="-O3 -fp-model precise -assume byterecl -heap-arrays -xCORE-AVX2 -axCORE-AVX512"
+###########################################################################
+
+$FFLAGS="-O3 -mtune=native"
+FFLAGS="-O3 -acc=gpu -gpu=cc60,cuda11.1 -Minfo=accel"
+
+###########################################################################
+###########################################################################
+###########################################################################
+
+POT3D_HOME=$PWD
+
+cd ${POT3D_HOME}/src
+cp Makefile.template Makefile
+sed -i "s#<FFLAGS>#${FFLAGS}#g" Makefile
+sed -i "s#<HDF5_INCLUDE_DIR>#${HDF5_INCLUDE_DIR}#g" Makefile
+sed -i "s#<HDF5_LIB_DIR>#${HDF5_LIB_DIR}#g" Makefile
+sed -i "s#<HDF5_LIB_FLAGS>#${HDF5_LIB_FLAGS}#g" Makefile
+make
+
+echo "cp ${POT3D_HOME}/src/pot3d ${POT3D_HOME}/bin/pot3d"
+
+cp ${POT3D_HOME}/src/pot3d ${POT3D_HOME}/bin/pot3d
+

clean.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+rm testsuite/validation/run/* 2>/dev/null
+rm bin/pot3d 2>/dev/null
+
+cd src/
+make clean 2>/dev/null
+rm Makefile 2>/dev/null
+
+
+
+
+
+
+

scripts/pot3d_validation.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+new_run=$1
+valid_run=$2
+
+cX="\033[0m"
+cG="\033[32m"
+echo="echo -e"
+
+br1=$(grep "Energy in Br" $1)
+br2=$(grep "Energy in Br" $2)
+
+bt1=$(grep "Energy in Bt" $1)
+bt2=$(grep "Energy in Bt" $2)
+
+bp1=$(grep "Energy in Bp" $1)
+bp2=$(grep "Energy in Bp" $2)
+
+if [ "$br1" == "" ]; then
+ ${echo} "Run may have ${cR}FAILED${cX} to run!"
+ exit 1
+fi
+
+passed=0
+
+if [ "$br1" != "$br2" ]; then
+ passed=$(($passed + 1))
+fi 
+
+if [ "$bt1" != "$bt2" ]; then
+ passed=$(($passed + 1))
+fi
+
+if [ "$bp1" != "$bp2" ]; then
+ passed=$(($passed + 1))
+fi
+
+if [ $passed == 0 ]; then
+ ${echo} "Run has ${cG}PASSED${cX} validation!"
+else
+ ${echo} "Run may have ${cR}FAILED${cX} validation!"
+ echo " "
+ echo "RUN1 BR: $br1"
+ echo "RUN2 BR: $br1"
+ echo " "
+ echo "RUN1 BT: $bt1"
+ echo "RUN2 BT: $bt1"
+ echo " "
+ echo "RUN1 BP: $bp1"
+ echo "RUN2 BP: $bp1"
+fi
+
+exit 0

src/Makefile.template

@@ -0,0 +1,39 @@
+FC = mpif90
+
+FFLAGS = <FFLAGS> -I<HDF5_INCLUDE_DIR>
+
+OBJS = number_types.o \
+ zm_parse_modules.o \
+ zm_parse.o \
+ zm_sds_modules.o \
+ zm_sds.o \
+ pot3d.o
+
+LDFLAGS = -L<HDF5_LIB_DIR> <HDF5_LIB_FLAGS>
+
+all: $(OBJS)
+ $(FC) $(FFLAGS) $(OBJS) $(LDFLAGS) -o pot3d
+ rm *.mod *.o 2>/dev/null
+
+clean:
+ rm pot3d 2>/dev/null
+ rm -f *.mod *.o 2>/dev/null
+
+number_types.o: number_types.f
+ $(FC) -c $(FFLAGS) $<
+
+zm_parse_modules.o: zm_parse_modules.f
+ $(FC) -c $(FFLAGS) $<
+
+zm_parse.o: zm_parse.f zm_parse_modules.f number_types.f
+ $(FC) -c $(FFLAGS) $<
+
+zm_sds_modules.o: zm_sds_modules.f
+ $(FC) -c $(FFLAGS) $<
+
+zm_sds.o: zm_sds.f zm_sds_modules.f number_types.f
+ $(FC) -c $(FFLAGS) $<
+
+pot3d.o: pot3d.f
+ $(FC) -c $(FFLAGS) $<
+

src/number_types.f

@@ -0,0 +1,22 @@
+c#######################################################################
+ module number_types
+c
+c-----------------------------------------------------------------------
+c ****** Basic number types.
+c ****** This module is used to set the default precision for REALs.
+c-----------------------------------------------------------------------
+c
+ use iso_fortran_env
+c
+c-----------------------------------------------------------------------
+c
+ implicit none
+c
+ integer, parameter :: KIND_REAL_4=REAL32
+ integer, parameter :: KIND_REAL_8=REAL64
+ integer, parameter :: KIND_REAL_16=max(REAL128,REAL64)
+c
+ integer, parameter :: r_typ=KIND_REAL_8
+c
+ end module
+c#######################################################################