Integration of ELPA for eigendecomposition of projected Hamiltonian. …

…Added relevant ctest. Next use of ELPA for spectrum splitting Rayleigh Ritz.
ferroteam · Nov 21, 2018 · d08eb44 · d08eb44
1 parent 622f0c1
commit d08eb44
Show file tree

Hide file tree

Showing 15 changed files with 625 additions and 30 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -213,7 +213,7 @@ ELSE()
 ENDIF (DEFINED LIBXC_DIR)
 
 #
-# XML. Set with -DXML_DIR=/path/to/xml
+# XML. Set with -DXML_LIB_DIR and -DXML_INCLUDE_DIR
 #
 IF (DEFINED XML_INCLUDE_DIR)
  FIND_LIBRARY(XML_LIBRARY
@@ -230,7 +230,7 @@ IF (DEFINED XML_INCLUDE_DIR)
  )
  INCLUDE_DIRECTORIES ("${XML_INCLUDE_DIR}")
 ELSE()
- MESSAGE(FATAL_ERROR "-- Provide path of XML: -DXML_DIR=/path/to/xml")
+ MESSAGE(FATAL_ERROR "-- Provide path of XML: -DXML_LIB_DIR and -DXML_INCLUDE_DIR")
 ENDIF (DEFINED XML_INCLUDE_DIR)
 
 
@@ -256,6 +256,37 @@ ELSE()
  MESSAGE(FATAL_ERROR "-- Provide path to prefix of SPGLIB: -DSPGLIB_DIR=/path/to/prefix")
 ENDIF (DEFINED SPGLIB_DIR)
 
+#
+# ELPA. Set with -DELPA_LIB_DIR and -DELPA_INCLUDE_DIR
+#
+IF (WITH_ELPA)
+ ADD_DEFINITIONS(-DDFTFE_WITH_ELPA)
+ IF(NOT DEAL_II_WITH_SCALAPACK)
+ MESSAGE(FATAL_ERROR "
+ Error! The deal.II library found at ${DEAL_II_PATH} was not configured with
+ DEAL_II_WITH_SCALAPACK = ON
+ which is required for using ELPA."
+ )
+ ENDIF()
+
+ IF (DEFINED ELPA_INCLUDE_DIR)
+ FIND_LIBRARY(ELPA_LIBRARY
+ NAMES elpa elpa_openmp
+ HINTS ${ELPA_LIB_DIR}
+ NO_DEFAULT_PATH
+ )
+ IF(ELPA_LIBRARY STREQUAL "ELPA_LIBRARY-NOTFOUND")
+ MESSAGE(FATAL_ERROR "-- ELPA was not found in ${ELPA_LIB_DIR}")
+ ENDIF()
+ MESSAGE("-- Use ELPA from ${ELPA_LIB_DIR}")
+ TARGET_LINK_LIBRARIES(${TARGETLIB}
+ "${ELPA_LIBRARY}"
+ )
+ INCLUDE_DIRECTORIES ("${ELPA_INCLUDE_DIR}")
+ ELSE()
+ MESSAGE(FATAL_ERROR "-- Provide path of ELPA: -DELPA_LIB_DIR and -DELPA_INCLUDE_DIR")
+ ENDIF (DEFINED ELPA_INCLUDE_DIR)
+ENDIF()
 
 #
 # Custom "debug" and "release" make targets:

diff --git a/helpers/UMFlux/setupFluxELPAForReal.sh b/helpers/UMFlux/setupFluxELPAForReal.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+set -e
+set -o pipefail
+#script to setup and build DFT-FE
+
+########################################################################
+#Provide paths below for external libraries, compiler options and flags,
+# and optimization flag
+
+#Paths for external libraries
+dealiiPetscRealDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel18.0.1_real_processGridOld"
+dealiiPetscComplexDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscComplex_avx_64bit_mklthread_scalapack"
+alglibDir="/nfs/mcfs_comp/home/rudraa/software/alglib/cpp/src"
+libxcDir="/home/vikramg/DFT-FE-softwares/softwareCentos/libxcNew/install_intel18"
+spglibDir="/home/vikramg/DFT-FE-softwares/softwareCentos/spglib"
+xmlIncludeDir="/usr/include/libxml2"
+xmlLibDir="/usr/lib64"
+elpaIncludeDir="/home/vikramg/DFT-FE-softwares/softwareCentos/elpa/install2/include/elpa_openmp-2018.05.001"
+elpaLibDir="/home/vikramg/DFT-FE-softwares/softwareCentos/elpa/install2/lib"
+
+#If you have installed dealii by linking with intel mkl library set underlying flag to "ON",
+#otherwise set it to "OFF"
+withIntelMkl=ON
+
+#Compiler options and flags
+c_compiler=mpicc
+cxx_compiler=mpicxx
+c_flagsRelease="-O3"
+cxx_flagsRelease="-O3"
+
+#Option to link to ELPA
+withELPA=ON
+
+#Optmization flag: 1 for optimized mode and 0 for debug mode compilation
+optimizedMode=1
+
+###########################################################################
+#Usually, no changes are needed below this line
+#
+RCol='\e[0m'
+Blu='\e[0;34m';
+if [ $optimizedMode == 1 ]; then
+ if [ -d "build/release" ]; then
+ echo -e "${Blu}build/release directory already present${RCol}"
+ # Control will enter here if build directory exists.
+ cd build
+ cd release
+ echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
+ echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
+ else
+ rm -rf build/release
+ echo -e "${Blu}Creating build directory...${RCol}"
+ mkdir -p build && cd build
+ mkdir -p release && cd release
+ echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
+ echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
+ fi
+else
+ if [ -d "build/debug" ]; then
+ echo -e "${Blu}build/debug directory already present${RCol}"
+ # Control will enter here if build directory exists.
+ cd build
+ cd debug
+ echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
+ echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
+ else
+ rm -rf build/debug
+ echo -e "${Blu}Creating build directory...${RCol}"
+ mkdir -p build && cd build
+ mkdir -p debug && cd debug
+ echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
+ echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
+ fi
+fi
+echo -e "${Blu}Build complete.${RCol}"
diff --git a/include/dftParameters.h b/include/dftParameters.h
@@ -75,6 +75,7 @@ namespace dftfe {
  extern unsigned int nbandGrps;
  extern bool computeEnergyEverySCF;
  extern unsigned int scalapackParalProcs;
+ extern unsigned int scalapackBlockSize;
  extern unsigned int natoms;
  extern unsigned int natomTypes;
  extern double lowerBoundUnwantedFracUpper;
@@ -89,6 +90,7 @@ namespace dftfe {
  extern unsigned int numAdaptiveFilterStates;
  extern bool useMixedPrecCheby;
  extern unsigned int spectrumSplitStartingScfIter;
+ extern bool useELPA;
 
  /**
  * Declare parameters.

diff --git a/include/linearAlgebraOperations.h b/include/linearAlgebraOperations.h
@@ -192,7 +192,8 @@ namespace dftfe
  * @return flag indicating success/failure. 1 for failure, 0 for success
  */
  template<typename T>
- unsigned int pseudoGramSchmidtOrthogonalization(std::vector<T> & X,
+ unsigned int pseudoGramSchmidtOrthogonalization(operatorDFTClass & operatorMatrix,
+ std::vector<T> & X,
  const unsigned int numberComponents,
  const MPI_Comm &interBandGroupComm,
  const MPI_Comm &mpiComm,

diff --git a/include/operator.h b/include/operator.h
@@ -23,7 +23,12 @@
 
 #include <headers.h>
 #include <constraintMatrixInfo.h>
-
+#ifdef DFTFE_WITH_ELPA
+extern "C"
+{
+#include <elpa/elpa.h>
+}
+#endif
 
 namespace dftfe{
 
@@ -44,6 +49,21 @@ namespace dftfe{
  */
  virtual ~operatorDFTClass() = 0;
 
+#ifdef DEAL_II_WITH_SCALAPACK
+ unsigned int getScalapackBlockSize() const;
+
+ void processGridOptionalELPASetup(const unsigned int na,
+ const unsigned int nev);
+
+#ifdef DFTFE_WITH_ELPA
+ void elpaUninit();
+
+ elpa_t & getElpaHandle();
+
+ elpa_autotune_t & getElpaAutoTuneHandle();
+#endif
+
+#endif
 
  /**
  * @brief initialize operatorClass
@@ -291,7 +311,49 @@ namespace dftfe{
  //mpi communicator
  //
  MPI_Comm d_mpi_communicator;
+
+#ifdef DEAL_II_WITH_SCALAPACK
+#ifdef DFTFE_WITH_ELPA
+ /// ELPA handle
+ elpa_t d_elpaHandle;
+
+ /// ELPA autotune handle
+ elpa_autotune_t d_elpaAutoTuneHandle;
+
+ /// processGrid mpi communicator
+ MPI_Comm d_processGridCommunicatorActive;
+#endif
+
+ //ScaLAPACK distributed format block size
+ unsigned int d_scalapackBlockSize;
+#endif
  };
 
+/*--------------------- Inline functions --------------------------------*/
+
+# ifndef DOXYGEN
+#ifdef DEAL_II_WITH_SCALAPACK
+ inline unsigned int
+ operatorDFTClass::getScalapackBlockSize() const
+ {
+ return d_scalapackBlockSize;
+ }
+
+#ifdef DFTFE_WITH_ELPA
+ inline
+ elpa_t & operatorDFTClass::getElpaHandle()
+ {
+ return d_elpaHandle;
+ }
+
+ inline
+ elpa_autotune_t & operatorDFTClass::getElpaAutoTuneHandle()
+ {
+ return d_elpaAutoTuneHandle;
+ }
+#endif
+#endif
+# endif // ifndef DOXYGEN
+
 }
 #endif
diff --git a/setup.sh b/setup.sh
@@ -8,13 +8,15 @@ set -o pipefail
 # and optimization flag
 
 #Paths for external libraries
-dealiiPetscRealDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscReal_avx_64bit_mklthread_scalapack"
+dealiiPetscRealDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel18.0.1_real_processGridOld"
 dealiiPetscComplexDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscComplex_avx_64bit_mklthread_scalapack"
 alglibDir="/nfs/mcfs_comp/home/rudraa/software/alglib/cpp/src"
 libxcDir="/home/vikramg/DFT-FE-softwares/softwareCentos/libxcNew/install_intel18"
 spglibDir="/home/vikramg/DFT-FE-softwares/softwareCentos/spglib"
 xmlIncludeDir="/usr/include/libxml2"
 xmlLibDir="/usr/lib64"
+elpaIncludeDir="/home/vikramg/DFT-FE-softwares/softwareCentos/elpa/install2/include/elpa_openmp-2018.05.001"
+elpaLibDir="/home/vikramg/DFT-FE-softwares/softwareCentos/elpa/install2/lib"
 
 #If you have installed dealii by linking with intel mkl library set underlying flag to "ON",
 #otherwise set it to "OFF"
@@ -23,8 +25,11 @@ withIntelMkl=ON
 #Compiler options and flags
 c_compiler=mpicc
 cxx_compiler=mpicxx
-c_flagsRelease=-O2
-cxx_flagsRelease=-O2
+c_flagsRelease="-O3"
+cxx_flagsRelease="-O3"
+
+#Option to link to ELPA
+withELPA=ON
 
 #Optmization flag: 1 for optimized mode and 0 for debug mode compilation
 optimizedMode=1
@@ -41,7 +46,7 @@ if [ $optimizedMode == 1 ]; then
  cd build
  cd release
  echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
  mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
  else
@@ -50,9 +55,9 @@ if [ $optimizedMode == 1 ]; then
  mkdir -p build && cd build
  mkdir -p release && cd release
  echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}"
- mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
+ mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl  ../../../. && make -j 4 && cd ../..
  fi
 else
  if [ -d "build/debug" ]; then
@@ -61,7 +66,7 @@ else
  cd build
  cd debug
  echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
  mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
  else
@@ -70,7 +75,7 @@ else
  mkdir -p build && cd build
  mkdir -p debug && cd debug
  echo -e "${Blu}Building Real executable in Debug mode...${RCol}"
- mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ..
+ mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl -DWITH_ELPA=$withELPA -DELPA_LIB_DIR=$elpaLibDir -DELPA_INCLUDE_DIR=$elpaIncludeDir ../../../. && make -j 4 && cd ..
  echo -e "${Blu}Building Complex executable in Debug mode...${RCol}"
  mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_BUILD_TYPE=Debug -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../..
  fi

diff --git a/src/dft/dft.cc b/src/dft/dft.cc
@@ -732,6 +732,12 @@ namespace dftfe {
  kohnShamDFTOperatorClass<FEOrder> kohnShamDFTEigenOperator(this,mpi_communicator);
  kohnShamDFTEigenOperator.init();
 
+#ifdef DEAL_II_WITH_SCALAPACK
+ kohnShamDFTEigenOperator.processGridOptionalELPASetup(d_numEigenValues,
+ d_numEigenValues);
+
+#endif
+
  if (dftParameters::verbosity>=4)
  dftUtils::printCurrentMemoryUsage(mpi_communicator,
  "Kohn-sham dft operator init called");
@@ -1664,6 +1670,12 @@ namespace dftfe {
  if (dftParameters::writeDensitySolutionFields)
  outputDensity();
 
+#ifdef DFTFE_WITH_ELPA
+ if (dftParameters::useELPA)
+ kohnShamDFTEigenOperator.elpaUninit();
+#endif
+
+
  if (dftParameters::verbosity>=1)
  pcout << std::endl<< "Elapsed wall time since start of the program: " << d_globalTimer.wall_time() << " seconds\n"<<std::endl;
  }