Merge pull request SPARC-X#195 from YaphetS-jx/dev

add GPU hook for SQ

ChangeLog

@@ -3,6 +3,12 @@
 -Name
 -changes
 
+--------------
+Sep 08, 2023
+Name: Xin Jing
+Changes: (highT/)
+1. Add GPU hook for SQ 
+
 --------------
 Sep 06, 2023
 Name: Tian Tian

src/forces.c

@@ -715,6 +715,15 @@ void Calculate_forces_xc_linear(SPARC_OBJ *pSPARC, double *forces_xc) {
 void Calculate_nonlocal_forces(SPARC_OBJ *pSPARC)
 {
  if (pSPARC->SQFlag == 1) {
+ #ifdef SPARCX_ACCEL
+ if (pSPARC->useACCEL == 1 && pSPARC->cell_typ == 0)
+ {
+ pSPARC->pSQ->forceFlag = 1;
+ double lambda_min, lambda_max;
+ ACCEL_SQ_LanczosAlgorithm_gauss(pSPARC, pSPARC->pSQ->DMnx_SQ, pSPARC->pSQ->DMny_SQ, pSPARC->pSQ->DMnz_SQ, &lambda_min, &lambda_max);
+ pSPARC->pSQ->forceFlag = 0;
+ } else
+ #endif
  Calculate_nonlocal_forces_SQ(pSPARC);
  } else if (pSPARC->isGammaPoint) {
  #ifdef SPARCX_ACCEL

src/highT/include/dssq.h

@@ -92,6 +92,7 @@ typedef struct _SQ_OBJ
  double **gwt;
  double *Veff_PR; // Veff operator within PR (process+Rcut) domain
  double *x_ex;
+ int forceFlag;
 } SQ_OBJ;
 
 

src/highT/sq.c

@@ -15,6 +15,11 @@
 #include <assert.h>
 #include <mpi.h>
 #include <time.h>
+#ifdef USE_MKL
+ #include <mkl.h>
+#else
+ #include <cblas.h>
+#endif
 
 #include "sq.h"
 #include "sqtool.h"
@@ -26,6 +31,9 @@
 #include "sqEnergy.h"
 #include "tools.h"
 
+#ifdef SPARCX_ACCEL
+#include "accel.h"
+#endif
 
 #define max(a,b) ((a)>(b)?(a):(b))
 #define min(a,b) ((a)<(b)?(a):(b))
@@ -85,17 +93,27 @@ void GaussQuadrature(SPARC_OBJ *pSPARC, int SCFCount) {
  time2 = MPI_Wtime();
  #endif
 
- t0 = (double *) malloc(sizeof(double) * pSQ->Nd_loc);
- int center = nloc[0] + nloc[1]*Nx_loc + nloc[2]*Nx_loc*Ny_loc; 
- for (nd = 0; nd < DMnd; nd ++) {
- // initialize t0 as identity vector
- memset(t0, 0, sizeof(double)*pSQ->Nd_loc);
- t0[center] = 1.0; 
-
- LanczosAlgorithm_gauss(pSPARC, t0, &lambda_min, &lambda_max, nd);
- pSQ->mineig[nd] = lambda_min;
- pSQ->maxeig[nd] = lambda_max;
- }
+ #ifdef SPARCX_ACCEL
+ if (pSPARC->useACCEL == 1 && pSPARC->cell_typ == 0)
+ {
+ ACCEL_SQ_LanczosAlgorithm_gauss(pSPARC, DMnx, DMny, DMnz, &lambda_min, &lambda_max);
+ }
+ else
+ #endif // SPARCX_ACCEL
+ {
+ t0 = (double *) malloc(sizeof(double) * pSQ->Nd_loc);
+ int center = nloc[0] + nloc[1]*Nx_loc + nloc[2]*Nx_loc*Ny_loc; 
+ for (nd = 0; nd < DMnd; nd ++) {
+ // initialize t0 as identity vector
+ memset(t0, 0, sizeof(double)*pSQ->Nd_loc);
+ t0[center] = 1.0; 
+
+ LanczosAlgorithm_gauss(pSPARC, t0, &lambda_min, &lambda_max, nd);
+ pSQ->mineig[nd] = lambda_min;
+ pSQ->maxeig[nd] = lambda_max;
+ }
+ free(t0);
+ }
 
  // This barrier fixed the severe slowdown of MPI communication on hive
  MPI_Barrier(pSQ->dmcomm_SQ); 
@@ -155,8 +173,7 @@ void GaussQuadrature(SPARC_OBJ *pSPARC, int SCFCount) {
  // Transfer rho into phi domain
  TransferDensity_sq2phi(pSPARC, rho, pSPARC->electronDens);
 
- free(rho);
- free(t0);
+ free(rho); 
 }
 
 

src/highT/sqInitialization.c

@@ -124,4 +124,5 @@ void init_SQ(SPARC_OBJ *pSPARC) {
  pSQ->DMVertices_PR[3], pSQ->DMVertices_PR[4], pSQ->DMVertices_PR[5],
  pSQ->nloc[0], pSQ->nloc[1], pSQ->nloc[2]);
 #endif
+ pSQ->forceFlag = 0;
 }

src/initialization.c

@@ -223,6 +223,15 @@ void Initialize(SPARC_OBJ *pSPARC, int argc, char *argv[]) {
  if (rank == 0) printf("\nrank = %d, Copying data from SPARC_Input into SPARC & set up subcomm took %.3f ms\n",rank,(t2-t1)*1000);
 #endif
 
+#ifdef SPARCX_ACCEL // Activating flag for using hardware acceleration at compile time.
+ pSPARC->useACCEL = 1;
+ if (rank == 0) 
+ { 
+ char *hwaccel[2] = { "DISABLED", "ENABLED" };
+ printf ("[INFO] Hardware acceleration is %s\n", hwaccel[pSPARC->useACCEL]);
+ }
+#endif // ACCEL
+
  // set up sub-communicators
  if (pSPARC->SQFlag == 1) {
  Setup_Comms_SQ(pSPARC);
@@ -240,22 +249,6 @@ void Initialize(SPARC_OBJ *pSPARC, int argc, char *argv[]) {
  }
  #endif
 
- // SPARCX_ACCEL_NOTE Need to add this. Make sure it is always within "#ifdef USE_DP_SUBEIG" branch
- // --- BEGIN. Alfredo Metere
- #ifdef ACCEL // Activating flag for using hardware acceleration at compile time.
- pSPARC->useACCEL = 1;
- // #else
- // pSPARC->useACCEL = 0;
-
-
- if (rank == 0) 
- { 
- char *hwaccel[2] = { "DISABLED", "ENABLED" };
- printf ("[INFO] Hardware acceleration is %s\n", hwaccel[pSPARC->useACCEL]);
- }
- #endif // ACCEL
- // --- END. Alfredo Metere
-
  pSPARC->DP_CheFSI = NULL;
  pSPARC->DP_CheFSI_kpt = NULL;
  if (pSPARC->isGammaPoint) init_DP_CheFSI(pSPARC);
@@ -3180,7 +3173,7 @@ void write_output_init(SPARC_OBJ *pSPARC) {
  }
 
  fprintf(output_fp,"***************************************************************************\n");
- fprintf(output_fp,"* SPARC (version Sep 05, 2023) *\n");
+ fprintf(output_fp,"* SPARC (version Sep 08, 2023) *\n");
  fprintf(output_fp,"* Copyright (c) 2020 Material Physics & Mechanics Group, Georgia Tech *\n");
  fprintf(output_fp,"* Distributed under GNU General Public License 3 (GPL) *\n");
  fprintf(output_fp,"* Start time: %s *\n",c_time_str);