From 2969eda6e6d8dfcd9f6823a99e8319e7339b8390 Mon Sep 17 00:00:00 2001 From: Bruce J Palmer Date: Thu, 14 Oct 2021 12:49:16 -0700 Subject: [PATCH 01/10] Reduced size of threaded tests so that they don't take as long. --- global/testing/thread_perf_contig.c | 2 +- global/testing/thread_perf_strided.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/global/testing/thread_perf_contig.c b/global/testing/thread_perf_contig.c index b9b6fcdb0..d3673b444 100644 --- a/global/testing/thread_perf_contig.c +++ b/global/testing/thread_perf_contig.c @@ -16,7 +16,7 @@ #define WARMUP 2 -#define DEFAULT_DIM 1024*1024 +#define DEFAULT_DIM 256*256 #define MAX_MESSAGE_SIZE DEFAULT_DIM*DEFAULT_DIM #define MAX_FACTOR 256 diff --git a/global/testing/thread_perf_strided.c b/global/testing/thread_perf_strided.c index 864e4e210..ba42b15c2 100644 --- a/global/testing/thread_perf_strided.c +++ b/global/testing/thread_perf_strided.c @@ -16,7 +16,7 @@ #define WARMUP 2 -#define DEFAULT_DIM 1024 +#define DEFAULT_DIM 256 #define MAX_MESSAGE_SIZE DEFAULT_DIM*DEFAULT_DIM #define MAX_FACTOR 256 From 4aceff77cdd189511b3fcc725018dda7eb4b09bf Mon Sep 17 00:00:00 2001 From: Bruce J Palmer Date: Fri, 29 Oct 2021 10:42:36 -0700 Subject: [PATCH 02/10] Adding test file to build --- global/testing/laplace.c | 464 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 global/testing/laplace.c diff --git a/global/testing/laplace.c b/global/testing/laplace.c new file mode 100644 index 000000000..0c5e52728 --- /dev/null +++ b/global/testing/laplace.c @@ -0,0 +1,464 @@ +#include +#include +#include + +#include "macdecls.h" +#include "ga.h" +#include "mp3.h" + +//#define WRITE_VTK +#define NDIM 128 + +/** + * Solve Laplace's equation on a cubic domain using the sparse matrix + * functionality in GA. + */ + +#define MAX_FACTOR 1024 +void grid_factor(int p, int xdim, int ydim, int zdim, + int *idx, int *idy, int *idz) { + int i, j, k; + int ip, ifac, pmax, prime[MAX_FACTOR]; + int fac[MAX_FACTOR]; + int ix, iy, iz, ichk; + + i = 1; +/** + * factor p completely + * first, find all prime numbers, besides 1, less than or equal to + * the square root of p + */ + ip = (int)(sqrt((double)p))+1; + pmax = 0; + for (i=2; i<=ip; i++) { + ichk = 1; + for (j=0; j MAX_FACTOR) printf("Overflow in grid_factor\n"); + prime[pmax-1] = i; + } + } +/** + * find all prime factors of p + */ + ip = p; + ifac = 0; + for (i=0; i= 0; i--) { + ix = xdim/(*idx); + iy = ydim/(*idy); + iz = zdim/(*idz); + if (ix >= iy && ix >= iz && ix > 1) { + *idx = fac[i]*(*idx); + } else if (iy >= ix && iy >= iz && iy > 1) { + *idy = fac[i]*(*idy); + } else if (iz >= ix && iz >= iy && iz > 1) { + *idz = fac[i]*(*idz); + } else { + printf("Too many processors in grid factoring routine\n"); + } + } +} + +int main(int argc, char **argv) { + int s_a, g_b, g_x, g_p, g_r, g_t; + int one; + int64_t one_64; + int me, nproc; + int idim, jdim, kdim; + int64_t xdim, ydim, zdim; + int64_t rdim, cdim, rdx, cdx; + int64_t ldx, ldxy; + int ipx, ipy, ipz, idx, idy, idz; + int64_t ilo, ihi, jlo, jhi, klo, khi; + int64_t i, j, k, ncnt; + int iproc, ld; + double x, y, z, val, h, rxdim, rydim, rzdim; + int64_t *ibuf, **iptr; + double *vptr; + double *vbuf; + int ok; + double one_r = 1.0; + double m_one_r = -1.0; + double ir, jr, ldr; + double xinc_p, yinc_p, zinc_p; + double xinc_m, yinc_m, zinc_m; + double alpha, beta, residual; + int nsave; + int heap=10000000, stack=10000000; + int iterations = 10000; + double tol, twopi; + FILE *PHI; + /* Intitialize a message passing library */ + one = 1; + one_64 = 1; + MP_INIT(argc,argv); + + /* Initialize GA */ + NGA_Initialize(); + + /* Interior points of the grid run from 0 to NDIM-1, boundary points are located + * at -1 and NDIM for each of the axes */ + idim = NDIM; + jdim = NDIM; + kdim = NDIM; + xdim = NDIM; + ydim = NDIM; + zdim = NDIM; + rxdim = 1.0; + rydim = 1.0; + rzdim = 1.0; + h = rxdim/((double)NDIM); + me = GA_Nodeid(); + nproc = GA_Nnodes(); + twopi = 8.0*atan(1.0); + + heap /= nproc; + stack /= nproc; + if(! MA_init(MT_F_DBL, stack, heap)) + GA_Error("MA_init failed",stack+heap); /* initialize memory allocator*/ + + /* factor array */ + grid_factor(nproc, idim, jdim, kdim, &ipx, &ipy, &ipz); + if (me == 0) { + printf("Solving Laplace's equation on %d processors\n",nproc); + printf("\n Using %d X %d X %d processor grid\n",ipx,ipy,ipz); + printf("\n Grid size is %d X %d X %d\n",idim,jdim,kdim); + } + /* figure out process location in proc grid */ + i = me; + idx = me%ipx; + i = (i-idx)/ipx; + idy = i%ipy; + idz = (i-idy)/ipy; + /* find bounding indices for this processor */ + ilo = (xdim*idx)/ipx; + if (idx < ipx-1) { + ihi = (xdim*(idx+1))/ipx-1; + } else { + ihi = xdim-1; + } + jlo = (ydim*idy)/ipy; + if (idy < ipy-1) { + jhi = (ydim*(idy+1))/ipy-1; + } else { + jhi = ydim-1; + } + klo = (zdim*idz)/ipz; + if (idz < ipz-1) { + khi = (zdim*(idz+1))/ipz-1; + } else { + khi = zdim-1; + } + + /* create sparse array */ + rdim = xdim*ydim*zdim; + cdim = xdim*ydim*zdim; + ldx = xdim; + ldxy = xdim*ydim; + s_a = NGA_Sprs_array_create64(rdim, cdim, C_DBL); + ncnt = 0; + /* Set elements of Laplace operator. Use a global indexing scheme and don't + * worry about setting elements locally. Count up values associated with + * boundaries */ + for (i=ilo; i<=ihi; i++) { + if (i == 0) { + xinc_m = 2.0; + xinc_p = 1.0; + } else if (i == NDIM - 1) { + xinc_m = 1.0; + xinc_p = 2.0; + } else { + xinc_p = 1.0; + xinc_m = 1.0; + } + for (j=jlo; j<=jhi; j++) { + if (j == 0) { + yinc_m = 2.0; + yinc_p = 1.0; + } else if (j == NDIM - 1) { + yinc_m = 1.0; + yinc_p = 2.0; + } else { + yinc_p = 1.0; + yinc_m = 1.0; + } + for (k=klo; k<=khi; k++) { + if (k == 0) { + zinc_m = 2.0; + zinc_p = 1.0; + } else if (k == NDIM - 1) { + zinc_m = 1.0; + zinc_p = 2.0; + } else { + zinc_p = 1.0; + zinc_m = 1.0; + } + rdx = i + j*ldx + k*ldxy; + val = -(xinc_p+xinc_m+yinc_p+yinc_m+zinc_p+zinc_m)/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,rdx,&val); + if (i+1 < xdim) { + cdx = i+1 + j*ldx + k*ldxy; + val = xinc_p/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,cdx,&val); + } else { + ncnt++; + } + if (i-1 >= 0) { + cdx = i-1 + j*ldx + k*ldxy; + val = xinc_m/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,cdx,&val); + } else { + ncnt++; + } + if (j+1 < ydim) { + cdx = i + (j+1)*ldx + k*ldxy; + val = yinc_p/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,cdx,&val); + } else { + ncnt++; + } + if (j-1 >= 0) { + cdx = i + (j-1)*ldx + k*ldxy; + val = yinc_m/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,cdx,&val); + } else { + ncnt++; + } + if (k+1 < zdim) { + cdx = i + j*ldx + (k+1)*ldxy; + val = zinc_p/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,cdx,&val); + } else { + ncnt++; + } + if (k-1 >= 0) { + cdx = i + j*ldx + (k-1)*ldxy; + val = zinc_m/(h*h); + NGA_Sprs_array_add_element64(s_a,rdx,cdx,&val); + } else { + ncnt++; + } + } + } + } + if (NGA_Sprs_array_assemble(s_a) && me == 0) { + printf("\n Sparse array assembly completed\n"); + } + + /* Construct RHS vector. Assume points on boundary are given by + * the equation f(x,y,z) = cos(twopi*x) + cos(twopi*y) + cos(twopi*z) */ + ibuf = (int64_t*)malloc(ncnt*sizeof(int64_t)); + iptr = (int64_t**)malloc(ncnt*sizeof(int64_t*)); + vbuf = (double*)malloc(ncnt*sizeof(double)); + for (i=0; i 1.0e-5 && ncnt < iterations) { + if (me==0) printf("Iteration: %d Tolerance: %e\n",(int)ncnt+1,tol); + NGA_Sprs_array_matvec_multiply(s_a, g_p, g_t); + alpha = GA_Ddot(g_t,g_p); + alpha = residual/alpha; + GA_Add(&one_r,g_x,&alpha,g_p,g_x); + alpha = -alpha; + GA_Add(&one_r,g_r,&alpha,g_t,g_r); + GA_Norm_infinity(g_r, &tol); + beta = residual; + residual = GA_Ddot(g_r,g_r); + beta = residual/beta; + GA_Add(&one_r,g_r,&beta,g_p,g_p); + ncnt++; + } + /* + if (me==0) printf("RHS Vector\n"); + GA_Print(g_b); + if (me==0) printf("Solution Vector\n"); + GA_Print(g_x); + */ + + if (ncnt == iterations) { + if (me==0) printf("Solution failed to converge\n"); + } else { + if (me==0) printf("Solution converged\n"); + } + + /* Write solution to file */ +#ifdef WRITE_VTK + if (me == 0) { + vbuf = (double*)malloc(xdim*ydim*sizeof(double)); + PHI = fopen("phi.vtk","w"); + fprintf(PHI,"# vtk DataFile Version 3.0\n"); + fprintf(PHI,"Laplace Equation Solution\n"); + fprintf(PHI,"ASCII\n"); + fprintf(PHI,"DATASET STRUCTURED_POINTS\n"); + fprintf(PHI,"DIMENSIONS %ld %ld %ld\n",xdim,ydim,zdim); + fprintf(PHI,"ORIGIN %12.6f %12.6f %12.6f\n",0.5*h,0.5*h,0.5*h); + fprintf(PHI,"SPACING %12.6f %12.6f %12.6f\n",h,h,h); + fprintf(PHI," \n"); + fprintf(PHI,"POINT_DATA %ld\n",xdim*ydim*zdim); + fprintf(PHI,"SCALARS Phi float\n"); + fprintf(PHI,"LOOKUP_TABLE default\n"); + for (k=0; k Date: Fri, 29 Oct 2021 10:43:01 -0700 Subject: [PATCH 03/10] Cleaned up header files. --- global/src/ga.h | 45 +++++++++++++++++------------------------ global/src/global.fh.in | 22 ++++++-------------- 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/global/src/ga.h b/global/src/ga.h index a5b11dc9d..d32341ca7 100644 --- a/global/src/ga.h +++ b/global/src/ga.h @@ -22,7 +22,6 @@ extern void GA_Add_constant_patch(int g,int *lo,int *hi,void *alpha); extern void GA_Add_diagonal(int g_a, int g_v); extern void GA_Add(void *alpha, int g_a, void* beta, int g_b, int g_c); extern int GA_Allocate(int g_a); -extern int GA_Deallocate(int g_a); extern int GA_Assemble_duplicate(int g_a, char *name, void *ptr); extern void GA_Brdcst(void *buf, int lenbuf, int root); extern SingleComplex GA_Cdot(int g_a, int g_b); @@ -105,13 +104,10 @@ extern int GA_Nnodes(void); extern int GA_Nodeid(void); extern void GA_Norm1(int g_a, double *nm); extern void GA_Norm_infinity(int g_a, double *nm); -extern int GA_Overlay(int g_a, int g_p); extern int GA_Pgroup_absolute_id(int pgroup, int pid); extern void GA_Pgroup_brdcst(int grp, void *buf, int lenbuf, int root); extern void GA_Pgroup_cgop(int grp, SingleComplex x[], int n, char *op); extern int GA_Pgroup_create(int *list, int count); -extern int GA_Pgroup_duplicate(int grp); -extern int GA_Pgroup_self(); extern int GA_Pgroup_destroy(int grp); extern void GA_Pgroup_dgop(int grp, double x[], int n, char *op); extern void GA_Pgroup_fgop(int grp, float x[], int n, char *op); @@ -145,6 +141,7 @@ extern void GA_Scan_copy(int g_a, int g_b, int g_sbit, int lo, int hi); extern void GA_Set_array_name(int g_a, char *name); extern void GA_Set_block_cyclic(int g_a, int dims[]); extern void GA_Set_block_cyclic_proc_grid(int g_a, int block[], int proc_grid[]); +extern void GA_Set_tiled_proc_grid(int g_a, int block[], int proc_grid[]); extern void GA_Set_chunk(int g_a, int chunk[]); extern void GA_Set_data(int g_a, int ndim, int dims[], int type); extern void GA_Set_debug(int flag); @@ -158,10 +155,7 @@ extern void GA_Set_pgroup(int g_a, int p_handle); extern void GA_Set_restricted(int g_a, int list[], int size); extern void GA_Set_restricted_range(int g_a, int lo_proc, int hi_proc); extern void GA_Set_property(int g_a, char *property); -extern void GA_Set_tiled_proc_grid(int g_a, int block[], int proc_grid[]); -extern void GA_Set_tiled_irreg_proc_grid(int g_a, int mapc[], int nblocks[], int proc_grid[]); extern void GA_Unset_property(int g_a); -extern void GA_Set_memory_dev(int g_a, char *device); extern void GA_Sgemm(char ta, char tb, int m, int n, int k, float alpha, int g_a, int g_b, float beta, int g_c ); extern void GA_Shift_diagonal(int g_a, void *c); extern int GA_Solve(int g_a, int g_b); @@ -183,7 +177,6 @@ extern int GA_Uses_ma(void); extern int GA_Uses_proc_grid(int g_a); extern int GA_Valid_handle(int g_a); extern int GA_Verify_handle(int g_a); -extern void GA_Version(int *major, int *minor, int *patch); extern double GA_Wtime(void); extern DoubleComplex GA_Zdot(int g_a, int g_b); extern void GA_Zgop(DoubleComplex x[], int n, char *op); @@ -199,7 +192,6 @@ extern void NGA_Access(int g_a, int lo[], int hi[], void *ptr, int ld[] extern void NGA_Acc(int g_a, int lo[], int hi[],void* buf,int ld[],void* alpha); extern void NGA_Add_patch(void * alpha, int g_a, int alo[], int ahi[], void * beta, int g_b, int blo[], int bhi[], int g_c, int clo[], int chi[]); extern int NGA_Allocate(int g_a); -extern int NGA_Deallocate(int g_a); extern void NGA_Alloc_gatscat_buf(int nelems); extern SingleComplex NGA_Cdot_patch(int g_a, char t_a, int alo[], int ahi[], int g_b, char t_b, int blo[], int bhi[]); extern int NGA_Compare_distr(int g_a, int g_b); @@ -224,7 +216,7 @@ extern int NGA_Duplicate(int g_a, char* array_name); extern void NGA_Error(char *str, int code); extern float NGA_Fdot_patch(int g_a, char t_a, int alo[], int ahi[], int g_b, char t_b, int blo[], int bhi[]); extern void NGA_Fence(void); -extern void NGA_Free_gatscat_buf(void); +extern void NGA_Free_gatscat_buf(); extern void NGA_Fill(int g_a, void *value); extern void NGA_Fill_patch(int g_a, int lo[], int hi[], void *val); extern void NGA_Gather(int g_a, void *v, int* subsArray[], int n); @@ -276,14 +268,11 @@ extern void NGA_NbWait(ga_nbhdl_t* nbhandle); extern int NGA_Ndim(int g_a); extern int NGA_Nnodes(void); extern int NGA_Nodeid(void); -extern int NGA_Overlay(int g_a, int g_p); extern void NGA_Periodic_acc(int g_a, int lo[], int hi[],void* buf,int ld[],void* alpha); extern void NGA_Periodic_get(int g_a, int lo[], int hi[], void* buf, int ld[]); extern void NGA_Periodic_put(int g_a, int lo[], int hi[], void* buf, int ld[]); extern int NGA_Pgroup_absolute_id(int pgroup, int pid); extern int NGA_Pgroup_create(int *list, int count); -extern int NGA_Pgroup_duplicate(int grp); -extern int NGA_Pgroup_self(); extern int NGA_Pgroup_destroy(int grp); extern int NGA_Pgroup_get_default(void); extern int NGA_Pgroup_get_mirror(void); @@ -322,6 +311,7 @@ extern void NGA_Select_elem(int g_a, char* op, void* val, int *index); extern void NGA_Set_array_name(int g_a, char *name); extern void NGA_Set_block_cyclic(int g_a, int dims[]); extern void NGA_Set_block_cyclic_proc_grid(int g_a, int block[], int proc_grid[]); +extern void NGA_Set_tiled_proc_grid(int g_a, int block[], int proc_grid[]); extern void NGA_Set_chunk(int g_a, int chunk[]); extern void NGA_Set_data(int g_a, int ndim, int dims[], int type); extern void NGA_Set_debug(int flag); @@ -330,13 +320,18 @@ extern void NGA_Set_irreg_distr(int g_a, int map[], int block[]); extern void NGA_Set_irreg_flag(int g_a, int flag); extern void NGA_Set_memory_limit(size_t limit); extern void NGA_Set_pgroup(int g_a, int p_handle); +extern void NGA_Set_property(int g_a, char *property); extern void NGA_Set_restricted(int g_a, int list[], int size); extern void NGA_Set_restricted_range(int g_a, int lo_proc, int hi_proc); -extern void NGA_Set_property(int g_a, char *property); -extern void NGA_Set_tiled_proc_grid(int g_a, int block[], int proc_grid[]); -extern void NGA_Set_tiled_irreg_proc_grid(int g_a, int mapc[], int nblocks[], int proc_grid[]); -extern void NGA_Unset_property(int g_a); -extern void NGA_Set_memory_dev(int g_a, char *device); +extern void NGA_Sprs_array_access_col_block(int s_a, int icol, int **idx, int **jdx, void *val); +extern void NGA_Sprs_array_access_col_block64(int s_a, int icol, long **idx, long **jdx, void *val); +extern void NGA_Sprs_array_add_element(int s_a, int idx, int jdx, void *val); +extern int NGA_Sprs_array_assemble(int s_a); +extern int NGA_Sprs_array_create(int idim, int jdim, int type); +extern void NGA_Sprs_array_column_distribution(int s_a, int iproc, int *lo, int *hi); +extern int NGA_Sprs_array_destroy(Integer s_a); +extern void NGA_Sprs_array_matvec_multiply(int s_a, int g_a, int g_v); +extern void NGA_Sprs_array_row_distribution(int s_a, int iproc, int *lo, int *hi); extern void NGA_Strided_acc(int g_a, int lo[], int hi[], int skip[], void* buf, int ld[], void *alpha); extern void NGA_Strided_get(int g_a, int lo[], int hi[], int skip[], void* buf, int ld[]); extern void NGA_Strided_put(int g_a, int lo[], int hi[], int skip[], void* buf, int ld[]); @@ -344,6 +339,7 @@ extern void NGA_Sync(void); extern void NGA_Terminate(void); extern int NGA_Total_blocks(int g_a); extern void NGA_Unlock(int mutex); +extern void NGA_Unset_property(int g_a); extern void NGA_Update_ghosts(int g_a); extern int NGA_Update_ghost_dir(int g_a, int dimension, int idir, int flag); extern void NGA_Update_ghosts_nb(int g_a, ga_nbhdl_t *nbhandle); @@ -351,7 +347,6 @@ extern int NGA_Uses_ma(void); extern int NGA_Uses_proc_grid(int g_a); extern int NGA_Valid_handle(int g_a); extern int NGA_Verify_handle(int g_a); -extern void NGA_Version(int *major, int *minor, int *patch); extern double NGA_Wtime(void); extern DoubleComplex NGA_Zdot_patch(int g_a, char t_a, int alo[], int ahi[], int g_b, char t_b, int blo[], int bhi[]); extern void NGA_Zero(int g_a); @@ -376,14 +371,10 @@ extern void GA_Patch_enum(int g_a, int lo, int hi, void *start, void *i extern void GA_Recip_patch64(int g_a,int64_t *lo, int64_t *hi); extern void GA_Scan_add64(int g_a, int g_b, int g_sbit, int64_t lo, int64_t hi, int excl); extern void GA_Scan_copy64(int g_a, int g_b, int g_sbit, int64_t lo, int64_t hi); -extern void GA_Set_block_cyclic64(int g_a, int64_t dims[]); -extern void GA_Set_block_cyclic_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); extern void GA_Set_chunk64(int g_a, int64_t chunk[]); extern void GA_Set_data64(int g_a, int ndim, int64_t dims[], int type); extern void GA_Set_ghosts64(int g_a, int64_t width[]); extern void GA_Set_irreg_distr64(int g_a, int64_t map[], int64_t block[]); -extern void GA_Set_tiled_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); -extern void GA_Set_tiled_irreg_proc_grid64(int g_a, int64_t mapc[], int64_t nblocks[], int64_t proc_grid[]); extern void GA_Sgemm64(char ta, char tb, int64_t m, int64_t n, int64_t k, float alpha, int g_a, int g_b, float beta, int g_c ); extern void GA_Step_bound_info_patch64(int g_xx, int64_t xxlo[], int64_t xxhi[], int g_vv, int64_t vvlo[], int64_t vvhi[], int g_xxll, int64_t xxlllo[], int64_t xxllhi[], int64_t g_xxuu, int64_t xxuulo[], int64_t xxuuhi[], void *boundmin, void *wolfemin, void *boundmax); extern void GA_Step_max_patch64(int g_a, int64_t alo[], int64_t ahi[], int g_b, int64_t blo[], int64_t bhi[], void *step); @@ -445,18 +436,18 @@ extern void NGA_Scatter_flat64(int g_a, void *v, int64_t subsArray[], i extern void NGA_Scatter_acc64(int g_a, void *v, int64_t* subsArray[], int64_t n, void *alpha); extern void NGA_Scatter_acc_flat64(int g_a, void *v, int64_t subsArray[], int64_t n, void *alpha); extern void NGA_Select_elem64(int g_a, char* op, void* val, int64_t* index); -extern void NGA_Set_block_cyclic64(int g_a, int64_t dims[]); -extern void NGA_Set_block_cyclic_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); extern void NGA_Set_data64(int g_a, int ndim, int64_t dims[], int type); extern void NGA_Set_ghosts64(int g_a, int64_t width[]); extern void NGA_Set_irreg_distr64(int g_a, int64_t map[], int64_t block[]); -extern void NGA_Set_tiled_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); -extern void NGA_Set_tiled_irreg_proc_grid64(int g_a, int64_t mapc[], int64_t nblocks[], int64_t proc_grid[]); extern void NGA_Strided_acc64(int g_a, int64_t lo[], int64_t hi[], int64_t skip[], void* buf, int64_t ld[], void *alpha); extern void NGA_Strided_get64(int g_a, int64_t lo[], int64_t hi[], int64_t skip[], void* buf, int64_t ld[]); extern void NGA_Strided_put64(int g_a, int64_t lo[], int64_t hi[], int64_t skip[], void* buf, int64_t ld[]); extern DoubleComplex NGA_Zdot_patch64(int g_a, char t_a, int64_t alo[], int64_t ahi[], int g_b, char t_b, int64_t blo[], int64_t bhi[]); extern void NGA_Zero_patch64(int g_a, int64_t lo[], int64_t hi[]); +extern int NGA_Sprs_array_create64(int64_t idim, int64_t jdim, int type); +extern void NGA_Sprs_array_add_element64(int s_a, int64_t idx, int64_t jdx, void *val); +extern void NGA_Sprs_array_row_distribution64(int s_a, int iproc, int64_t *lo, int64_t *hi); +extern void NGA_Sprs_array_column_distribution64(int s_a, int iproc, int64_t *lo, int64_t *hi); #ifdef __cplusplus } diff --git a/global/src/global.fh.in b/global/src/global.fh.in index f7820234d..b190827cf 100644 --- a/global/src/global.fh.in +++ b/global/src/global.fh.in @@ -3,7 +3,6 @@ parameter (ga_max_dim = GA_MAX_DIM) ! logical ga_allocate - logical ga_deallocate complex ga_cdot complex ga_cdot_patch integer ga_cluster_nnodes @@ -39,11 +38,8 @@ integer ga_ndim integer ga_nnodes integer ga_nodeid - logical ga_overlay integer ga_pgroup_absolute_id integer ga_pgroup_create - integer ga_pgroup_duplicate - integer ga_pgroup_self logical ga_pgroup_destroy integer ga_pgroup_get_default integer ga_pgroup_get_mirror @@ -74,7 +70,6 @@ double complex ga_zdot double complex ga_zdot_patch logical nga_allocate - logical nga_deallocate complex nga_cdot complex nga_cdot_patch integer nga_cluster_nnodes @@ -119,11 +114,8 @@ integer nga_ndim integer nga_nnodes integer nga_nodeid - logical nga_overlay integer nga_pgroup_absolute_id integer nga_pgroup_create - integer nga_pgroup_duplicate - integer nga_pgroup_self logical nga_pgroup_destroy integer nga_pgroup_get_default integer nga_pgroup_get_mirror @@ -140,6 +132,9 @@ logical nga_set_update5_info integer nga_solve integer nga_spd_invert + logical nga_sprs_array_assemble + logical nga_sprs_array_create + logical nga_sprs_array_destroy integer nga_total_blocks logical nga_update2_ghosts logical nga_update3_ghosts @@ -157,7 +152,6 @@ double complex nga_zdot_patch ! external ga_allocate - external ga_deallocate external ga_cdot external ga_cdot_patch external ga_cluster_nnodes @@ -193,11 +187,8 @@ external ga_ndim external ga_nnodes external ga_nodeid - external ga_overlay external ga_pgroup_absolute_id external ga_pgroup_create - external ga_pgroup_duplicate - external ga_pgroup_self external ga_pgroup_destroy external ga_pgroup_get_default external ga_pgroup_get_mirror @@ -228,7 +219,6 @@ external ga_zdot external ga_zdot_patch external nga_allocate - external nga_deallocate external nga_cdot external nga_cdot_patch external nga_cluster_nnodes @@ -276,11 +266,8 @@ external nga_ndim external nga_nnodes external nga_nodeid - external nga_overlay external nga_pgroup_absolute_id external nga_pgroup_create - external nga_pgroup_duplicate - external nga_pgroup_self external nga_pgroup_destroy external nga_pgroup_get_default external nga_pgroup_get_mirror @@ -298,6 +285,9 @@ external nga_set_update5_info external nga_solve external nga_spd_invert + external nga_sprs_array_assemble + external nga_sprs_array_create + external nga_sprs_array_destroy external nga_total_blocks external nga_update2_ghosts external nga_update3_ghosts From e40784564dcbc8d04b1e4e654fadb2518340d7fe Mon Sep 17 00:00:00 2001 From: Bruce J Palmer Date: Fri, 10 Dec 2021 14:02:29 -0800 Subject: [PATCH 04/10] Added some declarations so the ga++ interface builds properly. --- global/src/ga.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/global/src/ga.h b/global/src/ga.h index d32341ca7..f38bd9601 100644 --- a/global/src/ga.h +++ b/global/src/ga.h @@ -37,6 +37,7 @@ extern int GA_Compare_distr(int g_a, int g_b); extern void GA_Copy(int g_a, int g_b); extern int GA_Create_handle(void); extern int GA_Create_mutexes(int number); +extern int GA_Deallocate(int g_a); extern double GA_Ddot(int g_a, int g_b); extern void GA_Destroy(int g_a); extern int GA_Destroy_mutexes(void); @@ -104,12 +105,14 @@ extern int GA_Nnodes(void); extern int GA_Nodeid(void); extern void GA_Norm1(int g_a, double *nm); extern void GA_Norm_infinity(int g_a, double *nm); +extern int GA_Overlay(int g_a, int g_p); extern int GA_Pgroup_absolute_id(int pgroup, int pid); extern void GA_Pgroup_brdcst(int grp, void *buf, int lenbuf, int root); extern void GA_Pgroup_cgop(int grp, SingleComplex x[], int n, char *op); extern int GA_Pgroup_create(int *list, int count); extern int GA_Pgroup_destroy(int grp); extern void GA_Pgroup_dgop(int grp, double x[], int n, char *op); +extern int GA_Pgroup_duplicate(int grp); extern void GA_Pgroup_fgop(int grp, float x[], int n, char *op); extern int GA_Pgroup_get_default(void); extern int GA_Pgroup_get_mirror(void); @@ -119,6 +122,7 @@ extern void GA_Pgroup_lgop(int grp, long x[], int n, char *op); extern void GA_Pgroup_llgop(int grp, long long x[], int n, char *op); extern int GA_Pgroup_nnodes(int grp_id); extern int GA_Pgroup_nodeid(int grp_id); +extern int GA_Pgroup_self(); extern void GA_Pgroup_set_default(int p_handle); extern int GA_Pgroup_split(int grp_id, int num_group); extern int GA_Pgroup_split_irreg(int grp_id, int color); @@ -142,6 +146,7 @@ extern void GA_Set_array_name(int g_a, char *name); extern void GA_Set_block_cyclic(int g_a, int dims[]); extern void GA_Set_block_cyclic_proc_grid(int g_a, int block[], int proc_grid[]); extern void GA_Set_tiled_proc_grid(int g_a, int block[], int proc_grid[]); +extern void GA_Set_tiled_irreg_proc_grid(int g_a, int mapc[], int nblocks[], int proc_grid[]); extern void GA_Set_chunk(int g_a, int chunk[]); extern void GA_Set_data(int g_a, int ndim, int dims[], int type); extern void GA_Set_debug(int flag); @@ -151,6 +156,7 @@ extern void GA_Set_ghosts(int g_a, int width[]); extern void GA_Set_irreg_distr(int g_a, int map[], int block[]); extern void GA_Set_irreg_flag(int g_a, int flag); extern void GA_Set_memory_limit(size_t limit); +extern void GA_Set_memory_dev(int g_a, char *device); extern void GA_Set_pgroup(int g_a, int p_handle); extern void GA_Set_restricted(int g_a, int list[], int size); extern void GA_Set_restricted_range(int g_a, int lo_proc, int hi_proc); @@ -206,6 +212,7 @@ extern int NGA_Create(int type,int ndim,int dims[], char *name, int ch extern int NGA_Create_irreg_config(int type,int ndim,int dims[],char *name, int block[], int map[], int p_handle); extern int NGA_Create_irreg(int type,int ndim,int dims[],char *name, int block[], int map[]); extern int NGA_Create_handle(void); +extern int NGA_Deallocate(int g_a); extern int NGA_Deregister_type(int type); extern void NGA_Destroy(int g_a); extern int NGA_Destroy_mutexes(void); @@ -271,18 +278,27 @@ extern int NGA_Nodeid(void); extern void NGA_Periodic_acc(int g_a, int lo[], int hi[],void* buf,int ld[],void* alpha); extern void NGA_Periodic_get(int g_a, int lo[], int hi[], void* buf, int ld[]); extern void NGA_Periodic_put(int g_a, int lo[], int hi[], void* buf, int ld[]); +extern int NGA_Overlay(int g_a, int g_p); extern int NGA_Pgroup_absolute_id(int pgroup, int pid); extern int NGA_Pgroup_create(int *list, int count); extern int NGA_Pgroup_destroy(int grp); +extern void NGA_Pgroup_dgop(int grp, double x[], int n, char *op); +extern int NGA_Pgroup_duplicate(int grp); +extern void NGA_Pgroup_fgop(int grp, float x[], int n, char *op); extern int NGA_Pgroup_get_default(void); extern int NGA_Pgroup_get_mirror(void); extern int NGA_Pgroup_get_world(void); +extern void NGA_Pgroup_igop(int grp, int x[], int n, char *op); +extern void NGA_Pgroup_lgop(int grp, long x[], int n, char *op); +extern void NGA_Pgroup_llgop(int grp, long long x[], int n, char *op); extern int NGA_Pgroup_nnodes(int grp_id); extern int NGA_Pgroup_nodeid(int grp_id); +extern int NGA_Pgroup_self(); extern void NGA_Pgroup_set_default(int p_handle); extern int NGA_Pgroup_split(int grp_id, int num_group); extern int NGA_Pgroup_split_irreg(int grp_id, int color); extern void NGA_Pgroup_sync(int grp_id); +extern void NGA_Pgroup_zgop(int grp, DoubleComplex x[], int n, char *op); extern void NGA_Print_patch(int g_a, int lo[], int hi[], int pretty); extern void NGA_Proc_topology(int g_a, int proc, int coord[]); extern void NGA_Put(int g_a, int lo[], int hi[], void* buf, int ld[]); @@ -312,6 +328,7 @@ extern void NGA_Set_array_name(int g_a, char *name); extern void NGA_Set_block_cyclic(int g_a, int dims[]); extern void NGA_Set_block_cyclic_proc_grid(int g_a, int block[], int proc_grid[]); extern void NGA_Set_tiled_proc_grid(int g_a, int block[], int proc_grid[]); +extern void NGA_Set_tiled_irreg_proc_grid(int g_a, int mapc[], int nblocks[], int proc_grid[]); extern void NGA_Set_chunk(int g_a, int chunk[]); extern void NGA_Set_data(int g_a, int ndim, int dims[], int type); extern void NGA_Set_debug(int flag); @@ -319,6 +336,7 @@ extern void NGA_Set_ghosts(int g_a, int width[]); extern void NGA_Set_irreg_distr(int g_a, int map[], int block[]); extern void NGA_Set_irreg_flag(int g_a, int flag); extern void NGA_Set_memory_limit(size_t limit); +extern void NGA_Set_memory_dev(int g_a, char *device); extern void NGA_Set_pgroup(int g_a, int p_handle); extern void NGA_Set_property(int g_a, char *property); extern void NGA_Set_restricted(int g_a, int list[], int size); @@ -371,10 +389,14 @@ extern void GA_Patch_enum(int g_a, int lo, int hi, void *start, void *i extern void GA_Recip_patch64(int g_a,int64_t *lo, int64_t *hi); extern void GA_Scan_add64(int g_a, int g_b, int g_sbit, int64_t lo, int64_t hi, int excl); extern void GA_Scan_copy64(int g_a, int g_b, int g_sbit, int64_t lo, int64_t hi); +extern void GA_Set_block_cyclic64(int g_a, int64_t dims[]); +extern void GA_Set_block_cyclic_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); extern void GA_Set_chunk64(int g_a, int64_t chunk[]); extern void GA_Set_data64(int g_a, int ndim, int64_t dims[], int type); extern void GA_Set_ghosts64(int g_a, int64_t width[]); extern void GA_Set_irreg_distr64(int g_a, int64_t map[], int64_t block[]); +extern void GA_Set_tiled_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); +extern void GA_Set_tiled_irreg_proc_grid64(int g_a, int64_t mapc[], int64_t nblocks[], int64_t proc_grid[]); extern void GA_Sgemm64(char ta, char tb, int64_t m, int64_t n, int64_t k, float alpha, int g_a, int g_b, float beta, int g_c ); extern void GA_Step_bound_info_patch64(int g_xx, int64_t xxlo[], int64_t xxhi[], int g_vv, int64_t vvlo[], int64_t vvhi[], int g_xxll, int64_t xxlllo[], int64_t xxllhi[], int64_t g_xxuu, int64_t xxuulo[], int64_t xxuuhi[], void *boundmin, void *wolfemin, void *boundmax); extern void GA_Step_max_patch64(int g_a, int64_t alo[], int64_t ahi[], int g_b, int64_t blo[], int64_t bhi[], void *step); @@ -436,9 +458,13 @@ extern void NGA_Scatter_flat64(int g_a, void *v, int64_t subsArray[], i extern void NGA_Scatter_acc64(int g_a, void *v, int64_t* subsArray[], int64_t n, void *alpha); extern void NGA_Scatter_acc_flat64(int g_a, void *v, int64_t subsArray[], int64_t n, void *alpha); extern void NGA_Select_elem64(int g_a, char* op, void* val, int64_t* index); +extern void NGA_Set_block_cyclic64(int g_a, int64_t dims[]); +extern void NGA_Set_block_cyclic_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); extern void NGA_Set_data64(int g_a, int ndim, int64_t dims[], int type); extern void NGA_Set_ghosts64(int g_a, int64_t width[]); extern void NGA_Set_irreg_distr64(int g_a, int64_t map[], int64_t block[]); +extern void NGA_Set_tiled_proc_grid64(int g_a, int64_t block[], int64_t proc_grid[]); +extern void NGA_Set_tiled_irreg_proc_grid64(int g_a, int64_t mapc[], int64_t nblocks[], int64_t proc_grid[]); extern void NGA_Strided_acc64(int g_a, int64_t lo[], int64_t hi[], int64_t skip[], void* buf, int64_t ld[], void *alpha); extern void NGA_Strided_get64(int g_a, int64_t lo[], int64_t hi[], int64_t skip[], void* buf, int64_t ld[]); extern void NGA_Strided_put64(int g_a, int64_t lo[], int64_t hi[], int64_t skip[], void* buf, int64_t ld[]); From 89e38222c6e0e86d557491a771c21a86f2751f4d Mon Sep 17 00:00:00 2001 From: Bruce J Palmer Date: Fri, 10 Dec 2021 14:44:25 -0800 Subject: [PATCH 05/10] Fixed up some additional problems with compiling test suite. --- global/src/global.fh.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/global/src/global.fh.in b/global/src/global.fh.in index b190827cf..363398235 100644 --- a/global/src/global.fh.in +++ b/global/src/global.fh.in @@ -38,6 +38,7 @@ integer ga_ndim integer ga_nnodes integer ga_nodeid + logical ga_overlay integer ga_pgroup_absolute_id integer ga_pgroup_create logical ga_pgroup_destroy @@ -114,6 +115,7 @@ integer nga_ndim integer nga_nnodes integer nga_nodeid + logical nga_overlay integer nga_pgroup_absolute_id integer nga_pgroup_create logical nga_pgroup_destroy @@ -187,6 +189,7 @@ external ga_ndim external ga_nnodes external ga_nodeid + external ga_overlay external ga_pgroup_absolute_id external ga_pgroup_create external ga_pgroup_destroy @@ -266,6 +269,7 @@ external nga_ndim external nga_nnodes external nga_nodeid + external nga_overlay external nga_pgroup_absolute_id external nga_pgroup_create external nga_pgroup_destroy From b4b35820277f2b524ed077515595a4aa2d62d506 Mon Sep 17 00:00:00 2001 From: Ajay Panyala Date: Thu, 9 Jun 2022 15:15:04 -0700 Subject: [PATCH 06/10] update license files --- COPYRIGHT | 50 -------------------------------------------------- DISCLAIMER | 23 +++++++++++++++++++++++ LICENSE | 9 +++++++++ 3 files changed, 32 insertions(+), 50 deletions(-) delete mode 100644 COPYRIGHT create mode 100644 DISCLAIMER create mode 100644 LICENSE diff --git a/COPYRIGHT b/COPYRIGHT deleted file mode 100644 index 4ba515b02..000000000 --- a/COPYRIGHT +++ /dev/null @@ -1,50 +0,0 @@ -Copyright (c) 2006, - -Battelle Memorial Institute All rights reserved. - -1. Battelle Memorial Institute (hereinafter Battelle) hereby grants permission -to any person or entity lawfully obtaining a copy of this software and -associated documentation files (hereinafter "the Software") to redistribute -and use the Software in source and binary forms, with or without modification. -Such person or entity may use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and may permit others to do -so, subject to the following conditions: -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimers. -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. -* Other than as used herein, neither the name Battelle Memorial Institute or - Battelle may be used in any form whatsoever without the express written - consent of Battelle. - -2. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BATTELLE OR CONTRIBUTORS BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -3. The Software was produced by BMI under Contract No. DE-AC05-76RL01830 with -the Department of Energy. For five (5) years from February 23, 2006, the -Government is granted for itself and others acting on its behalf a -nonexclusive, paid-up, irrevocable worldwide license in this data to -reproduce, prepare derivative works, and perform publicly and display -publicly, by or on behalf of the Government. There is provision for the -possible extension of the term of this license. Subsequent to that period or -any extension granted, the Government is granted for itself and others acting -on its behalf a nonexclusive, paid-up, irrevocable worldwide license in this -data to reproduce, prepare derivative works, distribute copies to the public, -perform publicly and display publicly, and to permit others to do so. The -specific term of the license can be identified by inquiry made to Battelle or -DOE. Neither the United States nor the United States Department of Energy, -nor any of their employees, makes any warranty, express or implied, or assumes -any legal liability or responsibility for the accuracy, completeness or -usefulness of any data, apparatus, product or process disclosed, or represents -that its use would not infringe privately owned rights. - -Open Source License (0102) diff --git a/DISCLAIMER b/DISCLAIMER new file mode 100644 index 000000000..2b6ee327d --- /dev/null +++ b/DISCLAIMER @@ -0,0 +1,23 @@ +This material was prepared as an account of work sponsored by an agency of the +United States Government. Neither the United States Government nor the United +States Department of Energy, nor Battelle, nor any of their employees, nor any +jurisdiction or organization that has cooperated in the development of these +materials, makes any warranty, express or implied, or assumes any legal +liability or responsibility for the accuracy, completeness, or usefulness or +any information, apparatus, product, software, or process disclosed, or +represents that its use would not infringe privately owned rights. + +Reference herein to any specific commercial product, process, or service by +trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the United +States Government or any agency thereof, or Battelle Memorial Institute. The +views and opinions of authors expressed herein do not necessarily state or +reflect those of the United States Government or any agency thereof. + + PACIFIC NORTHWEST NATIONAL LABORATORY + operated by + BATTELLE + for the + UNITED STATES DEPARTMENT OF ENERGY + under Contract DE-AC05-76RL01830 + diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..40711a696 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright 2022 Battelle Memorial Institute. + +Contributed under the BSD-3-Clause License https://opensource.org/licenses/BSD-3-Clause + +By contributing computer software code to the GlobalArrays project I certify that the contribution +is being made under the BSD-3-Clause license (https://opensource.org/licenses/BSD-3-Clause) +and that I have all rights and permissions necessary to do so, including authorization from +my employer if applicable. + From 81c690b4244c7cc45fac5021f06a888e9f238930 Mon Sep 17 00:00:00 2001 From: Jeff Hammond Date: Wed, 7 Dec 2022 04:18:22 +0200 Subject: [PATCH 07/10] replace case-switch on MPI return codes with if-elseif (#286) error codes are not required to be compile-time constants. see issue and related links for details. fixes https://github.com/GlobalArrays/ga/issues/285 Signed-off-by: Jeff Hammond --- cmx/src-mpi-pr/cmx.c | 48 +++++++++++++++++--------------------- comex/src-mpi-mt/comex.c | 48 +++++++++++++++++--------------------- comex/src-mpi-pr/comex.c | 48 +++++++++++++++++--------------------- comex/src-mpi-pt/comex.c | 48 +++++++++++++++++--------------------- comex/src-portals4/comex.c | 48 +++++++++++++++++--------------------- 5 files changed, 110 insertions(+), 130 deletions(-) diff --git a/cmx/src-mpi-pr/cmx.c b/cmx/src-mpi-pr/cmx.c index e8a8884ec..8a3b231ca 100644 --- a/cmx/src-mpi-pr/cmx.c +++ b/cmx/src-mpi-pr/cmx.c @@ -5151,32 +5151,28 @@ STATIC void check_mpi_retval(int retval, const char *file, int line) STATIC const char *str_mpi_retval(int retval) { const char *msg = NULL; - - switch(retval) { - case MPI_SUCCESS : msg = "MPI_SUCCESS"; break; - case MPI_ERR_BUFFER : msg = "MPI_ERR_BUFFER"; break; - case MPI_ERR_COUNT : msg = "MPI_ERR_COUNT"; break; - case MPI_ERR_TYPE : msg = "MPI_ERR_TYPE"; break; - case MPI_ERR_TAG : msg = "MPI_ERR_TAG"; break; - case MPI_ERR_COMM : msg = "MPI_ERR_COMM"; break; - case MPI_ERR_RANK : msg = "MPI_ERR_RANK"; break; - case MPI_ERR_ROOT : msg = "MPI_ERR_ROOT"; break; - case MPI_ERR_GROUP : msg = "MPI_ERR_GROUP"; break; - case MPI_ERR_OP : msg = "MPI_ERR_OP"; break; - case MPI_ERR_TOPOLOGY : msg = "MPI_ERR_TOPOLOGY"; break; - case MPI_ERR_DIMS : msg = "MPI_ERR_DIMS"; break; - case MPI_ERR_ARG : msg = "MPI_ERR_ARG"; break; - case MPI_ERR_UNKNOWN : msg = "MPI_ERR_UNKNOWN"; break; - case MPI_ERR_TRUNCATE : msg = "MPI_ERR_TRUNCATE"; break; - case MPI_ERR_OTHER : msg = "MPI_ERR_OTHER"; break; - case MPI_ERR_INTERN : msg = "MPI_ERR_INTERN"; break; - case MPI_ERR_IN_STATUS : msg = "MPI_ERR_IN_STATUS"; break; - case MPI_ERR_PENDING : msg = "MPI_ERR_PENDING"; break; - case MPI_ERR_REQUEST : msg = "MPI_ERR_REQUEST"; break; - case MPI_ERR_LASTCODE : msg = "MPI_ERR_LASTCODE"; break; - default : msg = "DEFAULT"; break; - } - + if (retval == MPI_SUCCESS ) { msg = "MPI_SUCCESS"; } + else if (retval == MPI_ERR_BUFFER ) { msg = "MPI_ERR_BUFFER"; } + else if (retval == MPI_ERR_COUNT ) { msg = "MPI_ERR_COUNT"; } + else if (retval == MPI_ERR_TYPE ) { msg = "MPI_ERR_TYPE"; } + else if (retval == MPI_ERR_TAG ) { msg = "MPI_ERR_TAG"; } + else if (retval == MPI_ERR_COMM ) { msg = "MPI_ERR_COMM"; } + else if (retval == MPI_ERR_RANK ) { msg = "MPI_ERR_RANK"; } + else if (retval == MPI_ERR_ROOT ) { msg = "MPI_ERR_ROOT"; } + else if (retval == MPI_ERR_GROUP ) { msg = "MPI_ERR_GROUP"; } + else if (retval == MPI_ERR_OP ) { msg = "MPI_ERR_OP"; } + else if (retval == MPI_ERR_TOPOLOGY ) { msg = "MPI_ERR_TOPOLOGY"; } + else if (retval == MPI_ERR_DIMS ) { msg = "MPI_ERR_DIMS"; } + else if (retval == MPI_ERR_ARG ) { msg = "MPI_ERR_ARG"; } + else if (retval == MPI_ERR_UNKNOWN ) { msg = "MPI_ERR_UNKNOWN"; } + else if (retval == MPI_ERR_TRUNCATE ) { msg = "MPI_ERR_TRUNCATE"; } + else if (retval == MPI_ERR_OTHER ) { msg = "MPI_ERR_OTHER"; } + else if (retval == MPI_ERR_INTERN ) { msg = "MPI_ERR_INTERN"; } + else if (retval == MPI_ERR_IN_STATUS) { msg = "MPI_ERR_IN_STATUS"; } + else if (retval == MPI_ERR_PENDING ) { msg = "MPI_ERR_PENDING"; } + else if (retval == MPI_ERR_REQUEST ) { msg = "MPI_ERR_REQUEST"; } + else if (retval == MPI_ERR_LASTCODE ) { msg = "MPI_ERR_LASTCODE"; } + else { msg = "DEFAULT"; } return msg; } diff --git a/comex/src-mpi-mt/comex.c b/comex/src-mpi-mt/comex.c index a08ed1394..493bf8a93 100644 --- a/comex/src-mpi-mt/comex.c +++ b/comex/src-mpi-mt/comex.c @@ -2546,32 +2546,28 @@ STATIC void check_mpi_retval(int retval, const char *file, int line) STATIC const char *str_mpi_retval(int retval) { const char *msg = NULL; - - switch(retval) { - case MPI_SUCCESS : msg = "MPI_SUCCESS"; break; - case MPI_ERR_BUFFER : msg = "MPI_ERR_BUFFER"; break; - case MPI_ERR_COUNT : msg = "MPI_ERR_COUNT"; break; - case MPI_ERR_TYPE : msg = "MPI_ERR_TYPE"; break; - case MPI_ERR_TAG : msg = "MPI_ERR_TAG"; break; - case MPI_ERR_COMM : msg = "MPI_ERR_COMM"; break; - case MPI_ERR_RANK : msg = "MPI_ERR_RANK"; break; - case MPI_ERR_ROOT : msg = "MPI_ERR_ROOT"; break; - case MPI_ERR_GROUP : msg = "MPI_ERR_GROUP"; break; - case MPI_ERR_OP : msg = "MPI_ERR_OP"; break; - case MPI_ERR_TOPOLOGY : msg = "MPI_ERR_TOPOLOGY"; break; - case MPI_ERR_DIMS : msg = "MPI_ERR_DIMS"; break; - case MPI_ERR_ARG : msg = "MPI_ERR_ARG"; break; - case MPI_ERR_UNKNOWN : msg = "MPI_ERR_UNKNOWN"; break; - case MPI_ERR_TRUNCATE : msg = "MPI_ERR_TRUNCATE"; break; - case MPI_ERR_OTHER : msg = "MPI_ERR_OTHER"; break; - case MPI_ERR_INTERN : msg = "MPI_ERR_INTERN"; break; - case MPI_ERR_IN_STATUS : msg = "MPI_ERR_IN_STATUS"; break; - case MPI_ERR_PENDING : msg = "MPI_ERR_PENDING"; break; - case MPI_ERR_REQUEST : msg = "MPI_ERR_REQUEST"; break; - case MPI_ERR_LASTCODE : msg = "MPI_ERR_LASTCODE"; break; - default : msg = "DEFAULT"; break; - } - + if (retval == MPI_SUCCESS ) { msg = "MPI_SUCCESS"; } + else if (retval == MPI_ERR_BUFFER ) { msg = "MPI_ERR_BUFFER"; } + else if (retval == MPI_ERR_COUNT ) { msg = "MPI_ERR_COUNT"; } + else if (retval == MPI_ERR_TYPE ) { msg = "MPI_ERR_TYPE"; } + else if (retval == MPI_ERR_TAG ) { msg = "MPI_ERR_TAG"; } + else if (retval == MPI_ERR_COMM ) { msg = "MPI_ERR_COMM"; } + else if (retval == MPI_ERR_RANK ) { msg = "MPI_ERR_RANK"; } + else if (retval == MPI_ERR_ROOT ) { msg = "MPI_ERR_ROOT"; } + else if (retval == MPI_ERR_GROUP ) { msg = "MPI_ERR_GROUP"; } + else if (retval == MPI_ERR_OP ) { msg = "MPI_ERR_OP"; } + else if (retval == MPI_ERR_TOPOLOGY ) { msg = "MPI_ERR_TOPOLOGY"; } + else if (retval == MPI_ERR_DIMS ) { msg = "MPI_ERR_DIMS"; } + else if (retval == MPI_ERR_ARG ) { msg = "MPI_ERR_ARG"; } + else if (retval == MPI_ERR_UNKNOWN ) { msg = "MPI_ERR_UNKNOWN"; } + else if (retval == MPI_ERR_TRUNCATE ) { msg = "MPI_ERR_TRUNCATE"; } + else if (retval == MPI_ERR_OTHER ) { msg = "MPI_ERR_OTHER"; } + else if (retval == MPI_ERR_INTERN ) { msg = "MPI_ERR_INTERN"; } + else if (retval == MPI_ERR_IN_STATUS) { msg = "MPI_ERR_IN_STATUS"; } + else if (retval == MPI_ERR_PENDING ) { msg = "MPI_ERR_PENDING"; } + else if (retval == MPI_ERR_REQUEST ) { msg = "MPI_ERR_REQUEST"; } + else if (retval == MPI_ERR_LASTCODE ) { msg = "MPI_ERR_LASTCODE"; } + else { msg = "DEFAULT"; } return msg; } diff --git a/comex/src-mpi-pr/comex.c b/comex/src-mpi-pr/comex.c index c4c829348..327f7f722 100644 --- a/comex/src-mpi-pr/comex.c +++ b/comex/src-mpi-pr/comex.c @@ -4963,32 +4963,28 @@ STATIC void check_mpi_retval(int retval, const char *file, int line) STATIC const char *str_mpi_retval(int retval) { const char *msg = NULL; - - switch(retval) { - case MPI_SUCCESS : msg = "MPI_SUCCESS"; break; - case MPI_ERR_BUFFER : msg = "MPI_ERR_BUFFER"; break; - case MPI_ERR_COUNT : msg = "MPI_ERR_COUNT"; break; - case MPI_ERR_TYPE : msg = "MPI_ERR_TYPE"; break; - case MPI_ERR_TAG : msg = "MPI_ERR_TAG"; break; - case MPI_ERR_COMM : msg = "MPI_ERR_COMM"; break; - case MPI_ERR_RANK : msg = "MPI_ERR_RANK"; break; - case MPI_ERR_ROOT : msg = "MPI_ERR_ROOT"; break; - case MPI_ERR_GROUP : msg = "MPI_ERR_GROUP"; break; - case MPI_ERR_OP : msg = "MPI_ERR_OP"; break; - case MPI_ERR_TOPOLOGY : msg = "MPI_ERR_TOPOLOGY"; break; - case MPI_ERR_DIMS : msg = "MPI_ERR_DIMS"; break; - case MPI_ERR_ARG : msg = "MPI_ERR_ARG"; break; - case MPI_ERR_UNKNOWN : msg = "MPI_ERR_UNKNOWN"; break; - case MPI_ERR_TRUNCATE : msg = "MPI_ERR_TRUNCATE"; break; - case MPI_ERR_OTHER : msg = "MPI_ERR_OTHER"; break; - case MPI_ERR_INTERN : msg = "MPI_ERR_INTERN"; break; - case MPI_ERR_IN_STATUS : msg = "MPI_ERR_IN_STATUS"; break; - case MPI_ERR_PENDING : msg = "MPI_ERR_PENDING"; break; - case MPI_ERR_REQUEST : msg = "MPI_ERR_REQUEST"; break; - case MPI_ERR_LASTCODE : msg = "MPI_ERR_LASTCODE"; break; - default : msg = "DEFAULT"; break; - } - + if (retval == MPI_SUCCESS ) { msg = "MPI_SUCCESS"; } + else if (retval == MPI_ERR_BUFFER ) { msg = "MPI_ERR_BUFFER"; } + else if (retval == MPI_ERR_COUNT ) { msg = "MPI_ERR_COUNT"; } + else if (retval == MPI_ERR_TYPE ) { msg = "MPI_ERR_TYPE"; } + else if (retval == MPI_ERR_TAG ) { msg = "MPI_ERR_TAG"; } + else if (retval == MPI_ERR_COMM ) { msg = "MPI_ERR_COMM"; } + else if (retval == MPI_ERR_RANK ) { msg = "MPI_ERR_RANK"; } + else if (retval == MPI_ERR_ROOT ) { msg = "MPI_ERR_ROOT"; } + else if (retval == MPI_ERR_GROUP ) { msg = "MPI_ERR_GROUP"; } + else if (retval == MPI_ERR_OP ) { msg = "MPI_ERR_OP"; } + else if (retval == MPI_ERR_TOPOLOGY ) { msg = "MPI_ERR_TOPOLOGY"; } + else if (retval == MPI_ERR_DIMS ) { msg = "MPI_ERR_DIMS"; } + else if (retval == MPI_ERR_ARG ) { msg = "MPI_ERR_ARG"; } + else if (retval == MPI_ERR_UNKNOWN ) { msg = "MPI_ERR_UNKNOWN"; } + else if (retval == MPI_ERR_TRUNCATE ) { msg = "MPI_ERR_TRUNCATE"; } + else if (retval == MPI_ERR_OTHER ) { msg = "MPI_ERR_OTHER"; } + else if (retval == MPI_ERR_INTERN ) { msg = "MPI_ERR_INTERN"; } + else if (retval == MPI_ERR_IN_STATUS) { msg = "MPI_ERR_IN_STATUS"; } + else if (retval == MPI_ERR_PENDING ) { msg = "MPI_ERR_PENDING"; } + else if (retval == MPI_ERR_REQUEST ) { msg = "MPI_ERR_REQUEST"; } + else if (retval == MPI_ERR_LASTCODE ) { msg = "MPI_ERR_LASTCODE"; } + else { msg = "DEFAULT"; } return msg; } diff --git a/comex/src-mpi-pt/comex.c b/comex/src-mpi-pt/comex.c index 43b32f797..88c3ed412 100644 --- a/comex/src-mpi-pt/comex.c +++ b/comex/src-mpi-pt/comex.c @@ -3620,32 +3620,28 @@ STATIC void check_mpi_retval(int retval, const char *file, int line) STATIC const char *str_mpi_retval(int retval) { const char *msg = NULL; - - switch(retval) { - case MPI_SUCCESS : msg = "MPI_SUCCESS"; break; - case MPI_ERR_BUFFER : msg = "MPI_ERR_BUFFER"; break; - case MPI_ERR_COUNT : msg = "MPI_ERR_COUNT"; break; - case MPI_ERR_TYPE : msg = "MPI_ERR_TYPE"; break; - case MPI_ERR_TAG : msg = "MPI_ERR_TAG"; break; - case MPI_ERR_COMM : msg = "MPI_ERR_COMM"; break; - case MPI_ERR_RANK : msg = "MPI_ERR_RANK"; break; - case MPI_ERR_ROOT : msg = "MPI_ERR_ROOT"; break; - case MPI_ERR_GROUP : msg = "MPI_ERR_GROUP"; break; - case MPI_ERR_OP : msg = "MPI_ERR_OP"; break; - case MPI_ERR_TOPOLOGY : msg = "MPI_ERR_TOPOLOGY"; break; - case MPI_ERR_DIMS : msg = "MPI_ERR_DIMS"; break; - case MPI_ERR_ARG : msg = "MPI_ERR_ARG"; break; - case MPI_ERR_UNKNOWN : msg = "MPI_ERR_UNKNOWN"; break; - case MPI_ERR_TRUNCATE : msg = "MPI_ERR_TRUNCATE"; break; - case MPI_ERR_OTHER : msg = "MPI_ERR_OTHER"; break; - case MPI_ERR_INTERN : msg = "MPI_ERR_INTERN"; break; - case MPI_ERR_IN_STATUS : msg = "MPI_ERR_IN_STATUS"; break; - case MPI_ERR_PENDING : msg = "MPI_ERR_PENDING"; break; - case MPI_ERR_REQUEST : msg = "MPI_ERR_REQUEST"; break; - case MPI_ERR_LASTCODE : msg = "MPI_ERR_LASTCODE"; break; - default : msg = "DEFAULT"; break; - } - + if (retval == MPI_SUCCESS ) { msg = "MPI_SUCCESS"; } + else if (retval == MPI_ERR_BUFFER ) { msg = "MPI_ERR_BUFFER"; } + else if (retval == MPI_ERR_COUNT ) { msg = "MPI_ERR_COUNT"; } + else if (retval == MPI_ERR_TYPE ) { msg = "MPI_ERR_TYPE"; } + else if (retval == MPI_ERR_TAG ) { msg = "MPI_ERR_TAG"; } + else if (retval == MPI_ERR_COMM ) { msg = "MPI_ERR_COMM"; } + else if (retval == MPI_ERR_RANK ) { msg = "MPI_ERR_RANK"; } + else if (retval == MPI_ERR_ROOT ) { msg = "MPI_ERR_ROOT"; } + else if (retval == MPI_ERR_GROUP ) { msg = "MPI_ERR_GROUP"; } + else if (retval == MPI_ERR_OP ) { msg = "MPI_ERR_OP"; } + else if (retval == MPI_ERR_TOPOLOGY ) { msg = "MPI_ERR_TOPOLOGY"; } + else if (retval == MPI_ERR_DIMS ) { msg = "MPI_ERR_DIMS"; } + else if (retval == MPI_ERR_ARG ) { msg = "MPI_ERR_ARG"; } + else if (retval == MPI_ERR_UNKNOWN ) { msg = "MPI_ERR_UNKNOWN"; } + else if (retval == MPI_ERR_TRUNCATE ) { msg = "MPI_ERR_TRUNCATE"; } + else if (retval == MPI_ERR_OTHER ) { msg = "MPI_ERR_OTHER"; } + else if (retval == MPI_ERR_INTERN ) { msg = "MPI_ERR_INTERN"; } + else if (retval == MPI_ERR_IN_STATUS) { msg = "MPI_ERR_IN_STATUS"; } + else if (retval == MPI_ERR_PENDING ) { msg = "MPI_ERR_PENDING"; } + else if (retval == MPI_ERR_REQUEST ) { msg = "MPI_ERR_REQUEST"; } + else if (retval == MPI_ERR_LASTCODE ) { msg = "MPI_ERR_LASTCODE"; } + else { msg = "DEFAULT"; } return msg; } diff --git a/comex/src-portals4/comex.c b/comex/src-portals4/comex.c index 1c7542de4..717928bd6 100644 --- a/comex/src-portals4/comex.c +++ b/comex/src-portals4/comex.c @@ -1434,32 +1434,28 @@ static inline void check_mpi_retval(int retval, const char *file, int line) static inline const char *str_mpi_retval(int retval) { const char *msg = NULL; - - switch(retval) { - case MPI_SUCCESS : msg = "MPI_SUCCESS"; break; - case MPI_ERR_BUFFER : msg = "MPI_ERR_BUFFER"; break; - case MPI_ERR_COUNT : msg = "MPI_ERR_COUNT"; break; - case MPI_ERR_TYPE : msg = "MPI_ERR_TYPE"; break; - case MPI_ERR_TAG : msg = "MPI_ERR_TAG"; break; - case MPI_ERR_COMM : msg = "MPI_ERR_COMM"; break; - case MPI_ERR_RANK : msg = "MPI_ERR_RANK"; break; - case MPI_ERR_ROOT : msg = "MPI_ERR_ROOT"; break; - case MPI_ERR_GROUP : msg = "MPI_ERR_GROUP"; break; - case MPI_ERR_OP : msg = "MPI_ERR_OP"; break; - case MPI_ERR_TOPOLOGY : msg = "MPI_ERR_TOPOLOGY"; break; - case MPI_ERR_DIMS : msg = "MPI_ERR_DIMS"; break; - case MPI_ERR_ARG : msg = "MPI_ERR_ARG"; break; - case MPI_ERR_UNKNOWN : msg = "MPI_ERR_UNKNOWN"; break; - case MPI_ERR_TRUNCATE : msg = "MPI_ERR_TRUNCATE"; break; - case MPI_ERR_OTHER : msg = "MPI_ERR_OTHER"; break; - case MPI_ERR_INTERN : msg = "MPI_ERR_INTERN"; break; - case MPI_ERR_IN_STATUS : msg = "MPI_ERR_IN_STATUS"; break; - case MPI_ERR_PENDING : msg = "MPI_ERR_PENDING"; break; - case MPI_ERR_REQUEST : msg = "MPI_ERR_REQUEST"; break; - case MPI_ERR_LASTCODE : msg = "MPI_ERR_LASTCODE"; break; - default : msg = "DEFAULT"; break; - } - + if (retval == MPI_SUCCESS ) { msg = "MPI_SUCCESS"; } + else if (retval == MPI_ERR_BUFFER ) { msg = "MPI_ERR_BUFFER"; } + else if (retval == MPI_ERR_COUNT ) { msg = "MPI_ERR_COUNT"; } + else if (retval == MPI_ERR_TYPE ) { msg = "MPI_ERR_TYPE"; } + else if (retval == MPI_ERR_TAG ) { msg = "MPI_ERR_TAG"; } + else if (retval == MPI_ERR_COMM ) { msg = "MPI_ERR_COMM"; } + else if (retval == MPI_ERR_RANK ) { msg = "MPI_ERR_RANK"; } + else if (retval == MPI_ERR_ROOT ) { msg = "MPI_ERR_ROOT"; } + else if (retval == MPI_ERR_GROUP ) { msg = "MPI_ERR_GROUP"; } + else if (retval == MPI_ERR_OP ) { msg = "MPI_ERR_OP"; } + else if (retval == MPI_ERR_TOPOLOGY ) { msg = "MPI_ERR_TOPOLOGY"; } + else if (retval == MPI_ERR_DIMS ) { msg = "MPI_ERR_DIMS"; } + else if (retval == MPI_ERR_ARG ) { msg = "MPI_ERR_ARG"; } + else if (retval == MPI_ERR_UNKNOWN ) { msg = "MPI_ERR_UNKNOWN"; } + else if (retval == MPI_ERR_TRUNCATE ) { msg = "MPI_ERR_TRUNCATE"; } + else if (retval == MPI_ERR_OTHER ) { msg = "MPI_ERR_OTHER"; } + else if (retval == MPI_ERR_INTERN ) { msg = "MPI_ERR_INTERN"; } + else if (retval == MPI_ERR_IN_STATUS) { msg = "MPI_ERR_IN_STATUS"; } + else if (retval == MPI_ERR_PENDING ) { msg = "MPI_ERR_PENDING"; } + else if (retval == MPI_ERR_REQUEST ) { msg = "MPI_ERR_REQUEST"; } + else if (retval == MPI_ERR_LASTCODE ) { msg = "MPI_ERR_LASTCODE"; } + else { msg = "DEFAULT"; } return msg; } From befd78a2e6e095331a5ef60e87e35498997d4e10 Mon Sep 17 00:00:00 2001 From: Nick Wilson <12475993+nick-wilson@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:58:51 +0000 Subject: [PATCH 08/10] workaround MPI_COMM_NULL not available at compile time --- comex/src-mpi-pr/groups.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/comex/src-mpi-pr/groups.c b/comex/src-mpi-pr/groups.c index 3ec2bb687..ce377084a 100644 --- a/comex/src-mpi-pr/groups.c +++ b/comex/src-mpi-pr/groups.c @@ -26,6 +26,7 @@ /* world group state */ +#ifdef MPI_COMM_NULL comex_group_world_t g_state = { MPI_COMM_NULL, MPI_GROUP_NULL, @@ -37,6 +38,9 @@ comex_group_world_t g_state = { -1, -1 }; +#else +comex_group_world_t g_state = {0}; +#endif /* the HEAD of the group linked list */ comex_igroup_t *group_list = NULL; From 2fd652c766fc442bf75995d55ca7b8736e580c5b Mon Sep 17 00:00:00 2001 From: Nick Wilson <12475993+nick-wilson@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:59:12 +0000 Subject: [PATCH 09/10] workaround MPI_REQUEST_NULL not available at compile time --- comex/src-armci/message.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/comex/src-armci/message.c b/comex/src-armci/message.c index f1ffcb827..2334adda4 100644 --- a/comex/src-armci/message.c +++ b/comex/src-armci/message.c @@ -348,7 +348,11 @@ void armci_msg_brdcst(void* buffer, int len, int root) /* there was a case in ghost update where a proc sent a message to itself */ +#ifdef MPI_REQUEST_NULL static MPI_Request self_request = MPI_REQUEST_NULL; +#else +static MPI_Request self_request; +#endif static int self_request_flag = 0; void armci_msg_snd(int tag, void* buffer, int len, int to) From 6ea9364ea2518b453ed9293e0297e9e0c973bfb3 Mon Sep 17 00:00:00 2001 From: Nick Wilson <12475993+nick-wilson@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:59:35 +0000 Subject: [PATCH 10/10] workaround MPI_COMM_WORLD not available at compile time --- tcgmsg/tcgmsg-mpi/misc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tcgmsg/tcgmsg-mpi/misc.c b/tcgmsg/tcgmsg-mpi/misc.c index 91f1c48cc..64bba4e93 100644 --- a/tcgmsg/tcgmsg-mpi/misc.c +++ b/tcgmsg/tcgmsg-mpi/misc.c @@ -11,7 +11,11 @@ extern void exit(int status); #include "armci.h" char tcgmsg_err_string[ERR_STR_LEN]; +#ifdef MPI_COMM_WORLD MPI_Comm TCGMSG_Comm=MPI_COMM_WORLD; +#else +MPI_Comm TCGMSG_Comm; +#endif int _tcg_initialized=0; long DEBUG_; int SR_parallel; @@ -142,6 +146,9 @@ void tcgi_alt_pbegin(int *argc, char **argv[]) #else MPI_Init(argc, argv); #endif +#ifndef MPI_COMM_WORLD + TCGMSG_Comm=MPI_COMM_WORLD; +#endif #if defined(MPI_VERSION) && (MPI_VERSION >= 2) MPI_Comm_set_errhandler(TCGMSG_Comm, MPI_ERRORS_RETURN); #else