Merge branch 'chebyshevMemOpt' into pseudoGeneralize

Conflicts:
	setup.sh
	utils/dftParameters.cc

include/chebyshevOrthogonalizedSubspaceIterationSolver.h

@@ -57,7 +57,8 @@ namespace dftfe{
  vectorType & tempEigenVec,
  const unsigned int totalNumberWaveFunctions,
  std::vector<double> & eigenValues,
- std::vector<double> & residuals);
+ std::vector<double> & residuals,
+ const MPI_Comm &interBandGroupComm);
 
  /**
  * @brief Solve a generalized eigen problem.

include/dft.h

@@ -101,7 +101,9 @@ namespace dftfe {
  /**
  * dftClass constructor
  */
- dftClass(const MPI_Comm &mpi_comm_replica, const MPI_Comm &interpoolcomm);
+ dftClass(const MPI_Comm &mpi_comm_replica,
+ const MPI_Comm &interpoolcomm,
+ const MPI_Comm &interBandGroupComm);
  /**
  * dftClass destructor
  */
@@ -294,12 +296,13 @@ namespace dftfe {
  void computeNodalRhoFromQuadData();
 
  /**
- * sums rho cell quadratrure data from all kpoint pools
+ * sums rho cell quadratrure data from  inter communicator
  */
- void sumRhoDataKPointPools(std::map<dealii::CellId, std::vector<double> > * rhoValues,
- std::map<dealii::CellId, std::vector<double> > * gradRhoValues,
- std::map<dealii::CellId, std::vector<double> > * rhoValuesSpinPolarized,
- std::map<dealii::CellId, std::vector<double> > * gradRhoValuesSpinPolarized);
+ void sumRhoData(std::map<dealii::CellId, std::vector<double> > * rhoValues,
+ std::map<dealii::CellId, std::vector<double> > * gradRhoValues,
+ std::map<dealii::CellId, std::vector<double> > * rhoValuesSpinPolarized,
+ std::map<dealii::CellId, std::vector<double> > * gradRhoValuesSpinPolarized,
+ const MPI_Comm &interComm);
 
  /**
  * resize and allocate table storage for rho cell quadratrue data
@@ -458,7 +461,9 @@ namespace dftfe {
  /**
  * parallel objects
  */
- MPI_Comm mpi_communicator, interpoolcomm;
+ const MPI_Comm mpi_communicator;
+ const MPI_Comm interpoolcomm;
+ const MPI_Comm interBandGroupComm;
  const unsigned int n_mpi_processes;
  const unsigned int this_mpi_process;
  IndexSet locally_owned_dofs, locally_owned_dofsEigen;
@@ -627,9 +632,9 @@ namespace dftfe {
 
  /// k point weights
  std::vector<double> d_kPointWeights;
- 
+
  /// global k index of lower bound of the local k point set
- unsigned int lowerBoundKindex ; 
+ unsigned int lowerBoundKindex ;
  /**
  * Recomputes the k point cartesian coordinates from the crystal k point coordinates
  * and the current lattice vectors, which can change in each ground state solve when

include/dftParameters.h

@@ -72,6 +72,9 @@ namespace dftfe {
  extern unsigned int orthoRRWaveFuncBlockSize;
  extern unsigned int subspaceRotDofsBlockSize;
  extern bool enableSwitchToGS;
+ extern unsigned int nbandGrps;
+ extern bool computeEnergyEverySCF;
+ extern unsigned int scalapackParalProcs;
 
  /**
  * Declare parameters.

include/dftUtils.h

@@ -53,14 +53,26 @@ namespace dftfe {
  *
  * @param dataOut DataOut class object
  * @param intralpoolcomm mpi communicator of domain decomposition inside each pool
- * @param interpoolcomm mpi communicator across pools
+ * @param interpoolcomm mpi communicator across k point pools
+ * @param interBandGroupComm mpi communicator across band groups
  * @param fileName
  */
  void writeDataVTUParallelLowestPoolId(const dealii::DataOut<3> & dataOut,
  const MPI_Comm & intrapoolcomm,
  const MPI_Comm & interpoolcomm,
+ const MPI_Comm &interBandGroupComm,
  const std::string & fileName);
 
+ /** @brief Create index vector which is used for band parallelization
+ *
+ * @[in]param interBandGroupComm mpi communicator across band groups
+ * @[in]param numBands
+ * @[out]param bandGroupLowHighPlusOneIndices
+ */
+ void createBandParallelizationIndices(const MPI_Comm &interBandGroupComm,
+ const unsigned int numBands,
+ std::vector<unsigned int> & bandGroupLowHighPlusOneIndices);
+
  /**
  * A class to split the given communicator into a number of pools
  */

include/energyCalculator.h

@@ -35,10 +35,12 @@ namespace dftfe {
  * @brief Constructor
  *
  * @param mpi_comm mpi communicator of domain decomposition
- * @param interpool_comm mpi communicator of interpool communicator
+ * @param interpool_comm mpi interpool communicator over k points
+ * @param interBandGroupComm mpi interpool communicator over band groups
  */
  energyCalculator(const MPI_Comm &mpi_comm,
- const MPI_Comm &interpool_comm);
+ const MPI_Comm &interpool_comm,
+ const MPI_Comm &interBandGroupComm);
 
  /**
  * Computes total energy of the ksdft problem in the current state and also prints the
@@ -179,6 +181,7 @@ namespace dftfe {
 
  const MPI_Comm mpi_communicator;
  const MPI_Comm interpoolcomm;
+ const MPI_Comm interBandGroupComm;
 
  /// parallel message stream
  dealii::ConditionalOStream pcout;

include/linearAlgebraOperations.h

@@ -149,12 +149,14 @@ namespace dftfe
  * @param[in,out] X Given subspace as flattened array of multi-vectors.
  * In-place update of the given subspace
  * @param[in] numberComponents Number of multiple-fields
+ * @param[in] interBandGroupComm interpool communicator for parallelization over band groups
  *
  * @return flag indicating success/failure. 1 for failure, 0 for success
  */
  template<typename T>
  unsigned int pseudoGramSchmidtOrthogonalization(dealii::parallel::distributed::Vector<T> & X,
- const unsigned int numberComponents);
+ const unsigned int numberComponents,
+ const MPI_Comm &interBandGroupComm);
 
  /** @brief Compute Rayleigh-Ritz projection
  *
@@ -176,12 +178,14 @@ namespace dftfe
  * @param[in,out] X Given subspace as flattened array of multi-vectors.
  * In-place rotated subspace
  * @param[in] numberComponents Number of multiple-fields
+ * @param[in] interBandGroupComm interpool communicator for parallelization over band groups
  * @param[out] eigenValues of the Projected Hamiltonian
  */
  template<typename T>
  void rayleighRitz(operatorDFTClass & operatorMatrix,
  dealii::parallel::distributed::Vector<T> & X,
  const unsigned int numberComponents,
+ const MPI_Comm &interBandGroupComm,
  std::vector<double> & eigenValues);
 
  /** @brief Compute Compute residual norm associated with eigenValue problem of the given operator

include/linearAlgebraOperationsInternal.h

@@ -37,7 +37,6 @@ namespace dftfe
  */
  void createProcessGridSquareMatrix(const MPI_Comm & mpi_communicator,
  const unsigned size,
- const unsigned int rowsBlockSize,
  std::shared_ptr< const dealii::Utilities::MPI::ProcessGrid> & processGrid);
 
 
@@ -50,6 +49,16 @@ namespace dftfe
  std::map<unsigned int, unsigned int> & globalToLocalRowIdMap,
  std::map<unsigned int, unsigned int> & globalToLocalColumnIdMap);
 
+
+ /** @brief Mpi all reduce of ScaLAPACKMat across a given inter communicator.
+ * Used for band parallelization.
+ *
+ */
+ template<typename T>
+ void sumAcrossInterCommScaLAPACKMat(const std::shared_ptr< const dealii::Utilities::MPI::ProcessGrid> & processGrid,
+ dealii::ScaLAPACKMatrix<T> & mat,
+ const MPI_Comm &interComm);
+
  /** @brief Computes S=X^{T}*X and stores in a parallel ScaLAPACK matrix.
  * X^{T} is the subspaceVectorsArray in the column major format. S is the
  * overlapMatPar.
@@ -63,6 +72,7 @@ namespace dftfe
  void fillParallelOverlapMatrix(const dealii::parallel::distributed::Vector<T> & X,
  const unsigned int numberVectors,
  const std::shared_ptr< const dealii::Utilities::MPI::ProcessGrid> & processGrid,
+ const MPI_Comm &interBandGroupComm,
  dealii::ScaLAPACKMatrix<T> & overlapMatPar);
 
  /** @brief Computes X^{T}=Q*X^{T} inplace. X^{T} is the subspaceVectorsArray in the column major
@@ -77,6 +87,7 @@ namespace dftfe
  void subspaceRotation(dealii::parallel::distributed::Vector<T> & subspaceVectorsArray,
  const unsigned int numberSubspaceVectors,
  const std::shared_ptr< const dealii::Utilities::MPI::ProcessGrid> & processGrid,
+ const MPI_Comm &interBandGroupComm,
  const dealii::ScaLAPACKMatrix<T> & rotationMatPar,
  const bool rotationMatTranspose=false);
 

include/triangulationManager.h

@@ -45,10 +45,13 @@ namespace dftfe {
  public:
  /** @brief Constructor.
  *
- * @param mpi_comm_replica mpi_communicator of the current pool
- * @param interpoolcomm mpi_communicator across pools (required to synchronize mesh generation)
+ * @param mpi_comm_replica mpi_communicator of the current pool
+ * @param interpool_comm mpi interpool communicator over k points
+ * @param interBandGroupComm mpi interpool communicator over band groups
  */
- triangulationManager(const MPI_Comm &mpi_comm_replica,const MPI_Comm &interpoolcomm);
+ triangulationManager(const MPI_Comm &mpi_comm_replica,
+ const MPI_Comm &interpoolcomm,
+ const MPI_Comm &interBandGroupComm);
 
 
  /**
@@ -154,13 +157,17 @@ namespace dftfe {
  * @param [input]nComponents number of components of the dofHandler on which solution
  * vectors are based upon
  * @param [input]solutionVectors vector of parallel distributed solution vectors to be serialized
- * @param [input]interpoolComm interpool communicator to ensure serialization happens only in pool
+ * @param [input]interpoolComm This communicator is used to ensure serialization
+ * happens only in k point pool
+ * @param [input]interBandGroupComm This communicator to ensure serialization happens
+ * only in band group
  */
  void saveTriangulationsSolutionVectors
  (const unsigned int feOrder,
  const unsigned int nComponents,
  const std::vector< const dealii::parallel::distributed::Vector<double> * > & solutionVectors,
- const MPI_Comm & interpoolComm);
+ const MPI_Comm & interpoolComm,
+ const MPI_Comm &interBandGroupComm);
 
  /**
  * @brief de-serialize the triangulations and the associated solution vectors
@@ -180,11 +187,15 @@ namespace dftfe {
  * @brief serialize the triangulations and the associated cell quadrature data container
  *
  * @param [input]cellQuadDataContainerIn container of input cell quadrature data to be serialized
- * @param [input]interpoolComm interpool communicator to ensure serialization happens only in pool
+ * @param [input]interpoolComm This communicator is used to ensure serialization
+ * happens only in k point pool
+ * @param [input]interBandGroupComm This communicator to ensure serialization happens
+ * only in band group
  */
  void saveTriangulationsCellQuadData
  (const std::vector<const std::map<dealii::CellId, std::vector<double> > *> & cellQuadDataContainerIn,
- const MPI_Comm & interpoolComm);
+ const MPI_Comm & interpoolComm,
+ const MPI_Comm &interBandGroupComm);
 
  /**
  * @brief de-serialize the triangulations and the associated cell quadrature data container
@@ -264,6 +275,7 @@ namespace dftfe {
  //
  const MPI_Comm mpi_communicator;
  const MPI_Comm interpoolcomm;
+ const MPI_Comm interBandGroupComm;
  const unsigned int this_mpi_process;
  const unsigned int n_mpi_processes;
  dealii::ConditionalOStream pcout;

setup.sh

@@ -2,7 +2,12 @@
 set -e
 set -o pipefail
 #script to setup and build DFT-FE
-#Provide paths for external libraries and optimization flag (0 for Debug, 1 for Release)
+
+########################################################################
+#Provide paths below for external libraries, compiler options and flags,
+# and optimization flag
+
+#Paths for external libraries
 dealiiPetscRealDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscReal_avx_64bit_mklthread_scalapack"
 dealiiPetscComplexDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscComplex_avx_64bit_mklthread_scalapack"
 alglibDir="/nfs/mcfs_comp/home/rudraa/software/alglib/cpp/src"
@@ -11,8 +16,17 @@ spglibDir="/home/vikramg/DFT-FE-softwares/softwareCentos/spglib"
 xmlIncludeDir="/usr/include/libxml2"
 xmlLibDir="/usr/lib64"
 
+
+#Compiler options and flags
+c_compiler=mpicc
+cxx_compiler=mpicxx
+c_flagsRelease=-O2
+cxx_flagsRelease=-O2
+
+#Optmization flag: 1 for optimized mode and 0 for debug mode compilation
 optimizedMode=1
-#
+
+###########################################################################
 #Usually, no changes are needed below this line
 #
 RCol='\e[0m'
@@ -24,18 +38,18 @@ if [ $optimizedMode == 1 ]; then
  cd build
  cd release
  echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_CXX_FLAGS_RELEASE="-O3" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_C_FLAGS_RELEASE=$c_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
- mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_CXX_FLAGS_RELEASE="-O3" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ../..
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_C_FLAGS_RELEASE=$c_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ../..
  else
  rm -rf build/release
  echo -e "${Blu}Creating build directory...${RCol}"
  mkdir -p build && cd build
  mkdir -p release && cd release
  echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_CXX_FLAGS_RELEASE="-O3" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_C_FLAGS_RELEASE=$c_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
- mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_CXX_FLAGS_RELEASE="-O3" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ../..
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir  ../../../. && make -j 4 && cd ../..
  fi
 else
  if [ -d "build/debug" ]; then
@@ -44,18 +58,18 @@ else
  cd build
  cd debug
  echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
- mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ../..
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ../..
  else
  rm -rf build/debug
  echo -e "${Blu}Creating build directory...${RCol}"
  mkdir -p build && cd build
  mkdir -p debug && cd debug
  echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
- mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir  ../../../. && make -j 4 && cd ../..
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir ../../../. && make -j 4 && cd ../..
  fi
 fi
 echo -e "${Blu}Build complete.${RCol}"