diff --git a/setup.sh b/setup.sh index 04e9fbf1a..02609f12c 100755 --- a/setup.sh +++ b/setup.sh @@ -8,23 +8,23 @@ set -o pipefail # and optimization flag #Paths for external libraries -dealiiPetscRealDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscReal_avx_64bit_mklthread_scalapack" -dealiiPetscComplexDir="/home/vikramg/DFT-FE-softwares/softwareCentos/dealiiDev/intel_18.0.1_petscComplex_avx_64bit_mklthread_scalapack" -alglibDir="/nfs/mcfs_comp/home/rudraa/software/alglib/cpp/src" -libxcDir="/home/vikramg/DFT-FE-softwares/softwareCentos/libxcNew/install_intel18" -spglibDir="/home/vikramg/DFT-FE-softwares/softwareCentos/spglib" -xmlIncludeDir="/usr/include/libxml2" -xmlLibDir="/usr/lib64" +dealiiPetscRealDir="/home/iancclin/dftfe/dealii_real" +dealiiPetscComplexDir="/home/iancclin/dftfe/dealii_complex" +alglibDir="/home/iancclin/dftfe/alglib/cpp/src" +libxcDir="/home/iancclin/dftfe/libxc" +spglibDir="/home/iancclin/dftfe/spglib" +xmlIncludeDir="/home/iancclin/dftfe/libxml2/include/libxml2" +xmlLibDir="/home/iancclin/dftfe/libxml2/lib" #If you have installed dealii by linking with intel mkl library set underlying flag to "ON", #otherwise set it to "OFF" -withIntelMkl=ON +withIntelMkl=OFF #Compiler options and flags c_compiler=mpicc cxx_compiler=mpicxx -c_flagsRelease=-O2 -cxx_flagsRelease=-O2 +c_flagsRelease="-O2 -fpermissive" +cxx_flagsRelease="-O2 -fpermissive" #Optmization flag: 1 for optimized mode and 0 for debug mode compilation optimizedMode=1 @@ -41,18 +41,18 @@ if [ $optimizedMode == 1 ]; then cd build cd release echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_C_FLAGS_RELEASE=$c_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. + mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_C_FLAGS_RELEASE=$c_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. + mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. else rm -rf build/release echo -e "${Blu}Creating build directory...${RCol}" mkdir -p build && cd build mkdir -p release && cd release echo -e "${Blu}Building Real executable in Optimized (Release) mode...${RCol}" - mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_C_FLAGS_RELEASE=$c_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. + mkdir -p real && cd real && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_C_FLAGS_RELEASE="$c_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscRealDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd .. echo -e "${Blu}Building Complex executable in Optimized (Release) mode...${RCol}" - mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE=$cxx_flagsRelease -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. + mkdir -p complex && cd complex && cmake -DCMAKE_C_COMPILER=$c_compiler -DCMAKE_CXX_COMPILER=$cxx_compiler -DCMAKE_CXX_FLAGS_RELEASE="$cxx_flagsRelease" -DCMAKE_BUILD_TYPE=Release -DDEAL_II_DIR=$dealiiPetscComplexDir -DALGLIB_DIR=$alglibDir -DLIBXC_DIR=$libxcDir -DSPGLIB_DIR=$spglibDir -DXML_LIB_DIR=$xmlLibDir -DXML_INCLUDE_DIR=$xmlIncludeDir -DWITH_INTEL_MKL=$withIntelMkl ../../../. && make -j 4 && cd ../.. fi else if [ -d "build/debug" ]; then diff --git a/src/dft/dft.cc b/src/dft/dft.cc index ee18e8b2c..6b2cc9c7c 100644 --- a/src/dft/dft.cc +++ b/src/dft/dft.cc @@ -1475,11 +1475,16 @@ namespace dftfe { dealii::parallel::distributed::Vector rhoNodalField; matrix_free_data.initialize_dof_vector(rhoNodalField,densityDofHandlerIndex); rhoNodalField=0; + std::function::active_cell_iterator & cell , + const unsigned int q)> funcRho = + [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , + const unsigned int q) + {return (*rhoOutValues).find(cell->id())->second[q];}; dealii::VectorTools::project<3,dealii::parallel::distributed::Vector> (dealii::MappingQ1<3,3>(), dofHandler, constraintsNone, QGauss<3>(C_num1DQuad()), - [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , const unsigned int q) -> double {return (*rhoOutValues).find(cell->id())->second[q];}, + funcRho, rhoNodalField); rhoNodalField.update_ghost_values(); @@ -1489,22 +1494,32 @@ namespace dftfe { { matrix_free_data.initialize_dof_vector(rhoNodalFieldSpin0,densityDofHandlerIndex); rhoNodalFieldSpin0=0; + std::function::active_cell_iterator & cell , + const unsigned int q)> funcRhoSpin0 = + [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , + const unsigned int q) + {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q];}; dealii::VectorTools::project<3,dealii::parallel::distributed::Vector> (dealii::MappingQ1<3,3>(), dofHandler, constraintsNone, QGauss<3>(C_num1DQuad()), - [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , const unsigned int q) -> double {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q];}, + funcRhoSpin0, rhoNodalFieldSpin0); rhoNodalFieldSpin0.update_ghost_values(); matrix_free_data.initialize_dof_vector(rhoNodalFieldSpin1,densityDofHandlerIndex); rhoNodalFieldSpin1=0; + std::function::active_cell_iterator & cell , + const unsigned int q)> funcRhoSpin1 = + [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , + const unsigned int q) + {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q+1];}; dealii::VectorTools::project<3,dealii::parallel::distributed::Vector> (dealii::MappingQ1<3,3>(), dofHandler, constraintsNone, QGauss<3>(C_num1DQuad()), - [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , const unsigned int q) -> double {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q+1];}, + funcRhoSpin1, rhoNodalFieldSpin1); rhoNodalFieldSpin1.update_ghost_values(); } diff --git a/src/dft/initRho.cc b/src/dft/initRho.cc index 0cc250a25..1be8b4332 100644 --- a/src/dft/initRho.cc +++ b/src/dft/initRho.cc @@ -564,13 +564,21 @@ void dftClass::computeNodalRhoFromQuadData() // matrix_free_data.initialize_dof_vector(d_rhoNodalField,densityDofHandlerIndex); d_rhoNodalField=0; + + std::function::active_cell_iterator & cell , + const unsigned int q)> funcRho = + [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , + const unsigned int q) + {return (*rhoOutValues).find(cell->id())->second[q];}; + dealii::VectorTools::project<3,dealii::parallel::distributed::Vector> (dealii::MappingQ1<3,3>(), dofHandler, constraintsNone, QGauss<3>(C_num1DQuad()), - [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , const unsigned int q) -> double {return (*rhoOutValues).find(cell->id())->second[q];}, + funcRho, d_rhoNodalField); + d_rhoNodalField.update_ghost_values(); @@ -578,24 +586,40 @@ void dftClass::computeNodalRhoFromQuadData() { matrix_free_data.initialize_dof_vector(d_rhoNodalFieldSpin0,densityDofHandlerIndex); d_rhoNodalFieldSpin0=0; + + std::function::active_cell_iterator & cell , + const unsigned int q)> funcRhoSpin0 = + [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , + const unsigned int q) + {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q];}; + dealii::VectorTools::project<3,dealii::parallel::distributed::Vector> (dealii::MappingQ1<3,3>(), dofHandler, constraintsNone, QGauss<3>(C_num1DQuad()), - [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , const unsigned int q) -> double {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q];}, + funcRhoSpin0, d_rhoNodalFieldSpin0); + d_rhoNodalFieldSpin0.update_ghost_values(); matrix_free_data.initialize_dof_vector(d_rhoNodalFieldSpin1,densityDofHandlerIndex); d_rhoNodalFieldSpin1=0; + + std::function::active_cell_iterator & cell , + const unsigned int q)> funcRhoSpin1 = + [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , + const unsigned int q) + {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q+1];}; + dealii::VectorTools::project<3,dealii::parallel::distributed::Vector> (dealii::MappingQ1<3,3>(), dofHandler, constraintsNone, QGauss<3>(C_num1DQuad()), - [&](const typename dealii::DoFHandler<3>::active_cell_iterator & cell , const unsigned int q) -> double {return (*rhoOutValuesSpinPolarized).find(cell->id())->second[2*q+1];}, + funcRhoSpin1, d_rhoNodalFieldSpin1); + d_rhoNodalFieldSpin1.update_ghost_values(); } } diff --git a/src/triangulation/triangulationManager/restartUtils.cc b/src/triangulation/triangulationManager/restartUtils.cc index cb5b66655..92ef520df 100644 --- a/src/triangulation/triangulationManager/restartUtils.cc +++ b/src/triangulation/triangulationManager/restartUtils.cc @@ -225,46 +225,50 @@ namespace dftfe { } const unsigned int dataSizeInBytes=sizeof(double)*totalQuadVectorSize; - const unsigned int offset = d_parallelTriangulationUnmoved.register_data_attach - (dataSizeInBytes, - [&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, - const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, - void * data) -> void - { - if (cell->active() && cell->is_locally_owned()) - { - Assert((*cellQuadDataContainerIn[0]).find(cell->id())!=(*cellQuadDataContainerIn[0]).end(),ExcInternalError()); - - double* dataStore = reinterpret_cast(data); - - double tempArray[totalQuadVectorSize]; - unsigned int count=0; - for (unsigned int i=0; isecond.size(); - - for (unsigned int j=0; jid())->second[j]; - count++; - } - } - - std::memcpy(dataStore, - &tempArray[0], - dataSizeInBytes); - } - else - { - double* dataStore = reinterpret_cast(data); - double tempArray[totalQuadVectorSize]; - std::memcpy(dataStore, - &tempArray[0], - dataSizeInBytes); - } - } - ); + const std::function::cell_iterator &cell, + const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, + void * data)> funcSave = + [&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, + const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, + void * data) + { + if (cell->active() && cell->is_locally_owned()) + { + Assert((*cellQuadDataContainerIn[0]).find(cell->id())!=(*cellQuadDataContainerIn[0]).end(),ExcInternalError()); + + double* dataStore = reinterpret_cast(data); + + double tempArray[totalQuadVectorSize]; + unsigned int count=0; + for (unsigned int i=0; isecond.size(); + + for (unsigned int j=0; jid())->second[j]; + count++; + } + } + + std::memcpy(dataStore, + &tempArray[0], + dataSizeInBytes); + } + else + { + double* dataStore = reinterpret_cast(data); + double tempArray[totalQuadVectorSize]; + std::memcpy(dataStore, + &tempArray[0], + dataSizeInBytes); + } + }; + + + const unsigned int offset = d_parallelTriangulationUnmoved.register_data_attach(dataSizeInBytes, + funcSave); const std::string filename="parallelUnmovedTriaSolData.chk"; if (std::ifstream(filename) && this_mpi_process==0) @@ -306,63 +310,57 @@ namespace dftfe { //FIXME: The underlying function calls to register_data_attach to notify_ready_to_unpack //will need to re-evaluated after the dealii github issue #6223 is fixed - const unsigned int offset1 = d_parallelTriangulationMoved.register_data_attach - (totalQuadVectorSize*sizeof(double), - [&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, - const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, - void * data) -> void - { - } - ); - - d_parallelTriangulationMoved.notify_ready_to_unpack - (offset1,[&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, - const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, - const void * data) -> void - { - if (cell->active() && cell->is_locally_owned()) - { - const double* dataStore = reinterpret_cast(data); - - double tempArray[totalQuadVectorSize]; - - std::memcpy(&tempArray[0], - dataStore, - totalQuadVectorSize*sizeof(double)); - - unsigned int count=0; - for (unsigned int i=0; iid()]=std::vector(cellDataSizeContainer[i]); - for (unsigned int j=0; jid()][j]=tempArray[count]; - count++; - } - }//container loop - } - } - ); + std::function::cell_iterator &cell, + const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, + void * data)> dummyFunc = + [&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, + const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, + void * data) + {}; + const unsigned int offset1 = d_parallelTriangulationMoved.register_data_attach(totalQuadVectorSize*sizeof(double), + dummyFunc); + + + std::function::cell_iterator &cell, + const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, + void * data)> funcLoad = + [&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, + const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, + void * data) + { + if (cell->active() && cell->is_locally_owned()) + { + const double* dataStore = reinterpret_cast(data); + + double tempArray[totalQuadVectorSize]; + + std::memcpy(&tempArray[0], + dataStore, + totalQuadVectorSize*sizeof(double)); + + unsigned int count=0; + for (unsigned int i=0; iid()]=std::vector(cellDataSizeContainer[i]); + for (unsigned int j=0; jid()][j]=tempArray[count]; + count++; + } + }//container loop + } + }; + + d_parallelTriangulationMoved.notify_ready_to_unpack(offset1, + funcLoad); //dummy de-serialization for d_parallelTriangulationUnmoved to avoid assert fail in call to save //FIXME: This also needs to be re-evaluated after the dealii github issue #6223 is fixed - const unsigned int offset2 = d_parallelTriangulationUnmoved.register_data_attach - (totalQuadVectorSize*sizeof(double), - [&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, - const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, - void * data) -> void - { - } - ); - - d_parallelTriangulationUnmoved.notify_ready_to_unpack - (offset2,[&](const typename dealii::parallel::distributed::Triangulation<3>::cell_iterator &cell, - const typename dealii::parallel::distributed::Triangulation<3>::CellStatus status, - const void * data) -> void - { - } - ); + const unsigned int offset2 = d_parallelTriangulationUnmoved.register_data_attach(totalQuadVectorSize*sizeof(double), + dummyFunc); + d_parallelTriangulationUnmoved.notify_ready_to_unpack(offset2, + dummyFunc); } }