Feature/oshmem

.gitignore

@@ -1,3 +1,32 @@
+# Ignore build directories and their contents
+build_shmem/
+build/
+bld*/
+
+# Ignore CMake build directories and files
+CMakeFiles/
+cmake_install.cmake
+CMakeCache.txt
+CTestTestfile.cmake
+Makefile
+compile_commands.json
+globalarrays-config.cmake
+globalarrays-config-version.cmake
+globalarrays-targets.cmake
+
+# Ignore all files in any CMakeFiles directory recursively
+**/CMakeFiles/
+**/CMakeFiles/**
+
+# Ignore test and object files
+*.x
+*.o
+*.log
+
+# Ignore any .o, .x, .log files in subdirectories
+**/*.o
+**/*.x
+**/*.log
 # Compiled Object files
 *.slo
 *.lo

Makefile.am

@@ -88,6 +88,12 @@ endif
 
 LDADD += libga.la
 
+# When using OpenSHMEM, add MPI libraries explicitly for test programs
+# that call MPI functions (like ffflush.F with mpi_finalize)
+if WITH_OSHMEM
+LDADD += -lmpi_mpifh -lmpi
+endif
+
 # Certain trickery when turning Fortran support on or off.
 if ENABLE_F77
 MAYBE_FLIBS = $(FLIBS)
@@ -523,6 +529,7 @@ check_PROGRAMS += global/testing/testeig
 check_PROGRAMS += global/testing/testmatmult
 check_PROGRAMS += global/testing/testsolve
 check_PROGRAMS += global/testing/test
+check_PROGRAMS += global/testing/accTest
 check_PROGRAMS += global/testing/overlay
 check_PROGRAMS += global/testing/test_mirrored
 check_PROGRAMS += global/testing/types_test
@@ -721,6 +728,7 @@ GLOBAL_PARALLEL_TESTS += global/testing/testeig$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/testmatmult$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/testsolve$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/test$(EXEEXT)
+GLOBAL_PARALLEL_TESTS += global/testing/accTest$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/overlay$(EXEEXT)
 if HAVE_SCALAPACK
 GLOBAL_PARALLEL_TESTS += global/testing/testspd$(EXEEXT)
@@ -967,6 +975,7 @@ global_testing_sprs_testf_SOURCES          = global/testing/sprs_testf.F $(gtsrc
 global_testing_stride_SOURCES              = global/testing/stride.F $(gtsrcf)
 global_testing_testabstract_ops_SOURCES    = global/testing/testabstract_ops.c
 global_testing_test_SOURCES                = global/testing/test.F $(gtsrcf)
+global_testing_accTest_SOURCES             = global/testing/accTest.F $(gtsrcf)
 global_testing_test_mirrored_SOURCES       = global/testing/test.F $(gtsrcf)
 global_testing_overlay_SOURCES             = global/testing/overlay.F $(gtsrcf)
 global_testing_testc_SOURCES               = global/testing/testc.c
@@ -1134,7 +1143,8 @@ global/examples/boltzmann/printdat.F      \
 global/examples/boltzmann/properties.F    \
 global/examples/boltzmann/setup.F         \
 global/examples/boltzmann/timestep.F      \
-global/examples/boltzmann/vorticity.F
+global/examples/boltzmann/vorticity.F     \
+global/testing/ffflush.F
 global_examples_boltzmann_boltz_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/global/examples/boltzmann
 
 global_examples_conjugate_gradient_ga_cg_SOURCES = \

armci/src/xfer/strided.c

@@ -746,6 +746,14 @@ int PARMCI_GetS( void *src_ptr,  	/* pointer to 1st segment at source*/
 
   ORDER(GET,proc);
   ARMCI_INIT_HANDLE(&nbh);
+  /* Diagnostic: print seg_count and stride_levels to check units (bytes vs elements) */
+  {
+    int _i;
+    fprintf(stderr, "[ARMCI PARMCI_GetS] proc=%d stride_levels=%d seg_count:", proc, stride_levels);
+    for (_i = 0; _i <= stride_levels; _i++) fprintf(stderr, " %d", seg_count[_i]);
+    fprintf(stderr, "\n");
+    fflush(stderr);
+  }
   PARMCI_NbGetS(src_ptr,src_stride_arr,dst_ptr,dst_stride_arr,seg_count,stride_levels,proc,&nbh);
   PARMCI_Wait(&nbh);
   return 0;

comex/CMakeLists.txt

@@ -36,8 +36,64 @@ option (COMEX_NETWORK_MPI3 "use MPI RMA protocols for communication" OFF)
 option (COMEX_NETWORK_MPI_MT "use MPI multi-threading protocol for communication" OFF)
 option (COMEX_NETWORK_MPI_PT "use MPI progress threads protocol for communication" OFF)
 
+# Allow selecting an alternate runtime. When GA_RUNTIME is set to OPEN_SHMEM
+# the src-oshmem backend will be used instead of the MPI-based runtime.
+set(GA_RUNTIME "MPI" CACHE STRING "Runtime to build: MPI (default) or OPEN_SHMEM")
+set_property(CACHE GA_RUNTIME PROPERTY STRINGS MPI OPEN_SHMEM)
+
 include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR} src-common)
-if (MPI_TS)
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  set(COMEX_DEVICE
+    src-oshmem/comex.c
+    src-oshmem/groups.c
+      src-oshmem/nb.c
+      src-oshmem/locks.c
+  )
+  set (COMEX_NETWORK_OSHMEM ON)
+  include_directories(AFTER src-oshmem)
+  # Allow user to point to an OpenSHMEM installation root. If provided,
+  # we'll search under that prefix for headers and libraries.
+  option(OSHMEM_ROOT "Path to OpenSHMEM installation root" "")
+
+  # Find headers (shmem.h or openshmem.h) and library (libshmem / libopenshmem).
+  if (OSHMEM_ROOT)
+    set(_oshmem_hints ${OSHMEM_ROOT}/include ${OSHMEM_ROOT}/usr/include)
+    set(_oshmem_lib_hints ${OSHMEM_ROOT}/lib ${OSHMEM_ROOT}/lib64 ${OSHMEM_ROOT}/usr/lib ${OSHMEM_ROOT}/usr/lib64)
+  else()
+    set(_oshmem_hints /usr/include /usr/local/include)
+    set(_oshmem_lib_hints /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64 /opt/openmpi/lib)
+  endif()
+
+  find_path(SHMEM_INCLUDE
+    NAMES shmem.h openshmem.h
+    HINTS ${_oshmem_hints}
+  )
+
+  find_library(SHMEM_LIB
+    NAMES oshmem shmem openshmem liboshmem libshmem
+    HINTS ${_oshmem_lib_hints}
+  )
+
+  if (SHMEM_INCLUDE)
+    include_directories(AFTER ${SHMEM_INCLUDE})
+    message(STATUS "OpenSHMEM headers found: ${SHMEM_INCLUDE}")
+  else()
+    message(STATUS "OpenSHMEM headers NOT found; continuing but build may fail until OSHMEM_ROOT is set")
+  endif()
+
+  if (NOT SHMEM_LIB)
+    message(STATUS "OpenSHMEM library not found by CMake; will try linker name 'oshmem' then 'shmem' at link time")
+    # prefer 'oshmem' as the linker name for Open MPI's implementation
+    set(SHMEM_LIB oshmem)
+  else()
+    message(STATUS "OpenSHMEM library found: ${SHMEM_LIB}")
+  endif()
+  # Export the discovered library to the parent scope so top-level targets
+  # (e.g., the 'ga' library) can link against OpenSHMEM.
+  set(SHMEM_LIB ${SHMEM_LIB} CACHE STRING "OpenSHMEM library")
+  # Append to GA_EXTRA_LIBS in the parent scope so 'ga' picks it up
+  set(GA_EXTRA_LIBS ${GA_EXTRA_LIBS} ${SHMEM_LIB} PARENT_SCOPE)
+elseif (MPI_TS)
   set(COMEX_DEVICE
     src-mpi/comex.c
     src-mpi/groups.c
@@ -124,6 +180,10 @@ add_library(armci_comex OBJECT
 )
 target_compile_definitions(armci_comex PRIVATE HAVE_CONFIG_H)
 target_include_directories(armci_comex PRIVATE ${MPI_C_INCLUDE_DIRS})
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  # Ensure C sources see the OpenSHMEM build mode
+  target_compile_definitions(armci_comex PRIVATE _OPENSHMEM)
+endif()
 
 add_library(armci
   ${ARMCI_FILES}
@@ -134,6 +194,22 @@ target_compile_definitions(armci PRIVATE HAVE_CONFIG_H)
 add_library(GlobalArrays::armci ALIAS armci)
 
 target_include_directories(armci PRIVATE ${MPI_C_INCLUDE_DIRS})
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  target_compile_definitions(armci PRIVATE _OPENSHMEM)
+endif()
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  # Link OpenSHMEM into armci & comex libraries
+  target_link_libraries(armci PRIVATE ${SHMEM_LIB})
+endif()
+
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  # Also link the top-level 'ga' target if it exists so the final
+  # executable link commands get the SHMEM library. If 'ga' is not yet
+  # defined, we already appended SHMEM_LIB to GA_EXTRA_LIBS above.
+  if (TARGET ga)
+    target_link_libraries(ga PRIVATE ${SHMEM_LIB})
+  endif()
+endif()
 
 install(TARGETS armci
     EXPORT globalarrays-targets
@@ -148,6 +224,10 @@ add_library(comex
 add_library(GlobalArrays::comex ALIAS comex)
 
 target_include_directories(comex PRIVATE ${MPI_C_INCLUDE_DIRS})
+if (GA_RUNTIME STREQUAL "OPEN_SHMEM")
+  target_link_libraries(comex PRIVATE ${SHMEM_LIB})
+  target_compile_definitions(comex PRIVATE _OPENSHMEM)
+endif()
 
 install(TARGETS comex
     EXPORT globalarrays-targets

comex/Makefile.am

@@ -35,10 +35,15 @@ AM_LDFLAGS =
 LDADD =
 
 lib_LTLIBRARIES += libcomex.la
-# later Makefile fragments append to this
-libcomex_la_SOURCES =
+if COMEX_NETWORK_OSHMEM
+libcomex_la_SOURCES = src-oshmem/comex.c src-oshmem/groups.c src-oshmem/locks.c src-oshmem/nb.c
+else
+libcomex_la_SOURCES = src-mpi/comex.c src-mpi/groups.c
+endif
 libcomex_la_LIBADD =
+if !COMEX_NETWORK_OSHMEM
 libcomex_la_LIBADD += $(MPI_LIBS)
+endif
 libcomex_la_LIBADD += $(COMEX_NETWORK_LIBS)
 libcomex_la_LIBADD += $(BLAS_LIBS)
 libcomex_la_LIBADD += $(LIBS)
@@ -47,12 +52,16 @@ libcomex_la_LIBADD += $(LIBADD_DLOPEN)
 endif
 include_HEADERS += $(top_srcdir)/src-common/comex.h
 
+if !COMEX_NETWORK_OSHMEM
 AM_CPPFLAGS += $(MPI_CPPFLAGS)
+endif
 AM_CPPFLAGS += $(COMEX_NETWORK_CPPFLAGS)
 AM_CPPFLAGS += $(BLAS_CPPFLAGS)
 AM_CPPFLAGS += -I$(top_srcdir)/src-common
 
+if !COMEX_NETWORK_OSHMEM
 AM_LDFLAGS += $(MPI_LDFLAGS)
+endif
 AM_LDFLAGS += $(COMEX_NETWORK_LDFLAGS)
 AM_LDFLAGS += $(BLAS_LDFLAGS)
 
@@ -62,7 +71,9 @@ LDADD += libcomex.la
 noinst_LTLIBRARIES += libcomexi.la
 libcomexi_la_SOURCES = $(libcomex_la_SOURCES)
 libcomexi_la_LIBADD =
+if !COMEX_NETWORK_OSHMEM
 libcomexi_la_LIBADD += $(MPI_LIBS)
+endif
 libcomexi_la_LIBADD += $(COMEX_NETWORK_LIBS)
 libcomexi_la_LIBADD += $(BLAS_LIBS)
 libcomexi_la_LIBADD += $(LIBS)
@@ -143,6 +154,13 @@ if COMEX_NETWORK_MPI3
 include $(top_srcdir)/src-mpi3/Makefile.inc
 endif
 
+##############################################################################
+# src-oshmem (OpenSHMEM runtime)
+##############################################################################
+if COMEX_NETWORK_OSHMEM
+include $(top_srcdir)/src-oshmem/Makefile.inc
+endif
+
 ##############################################################################
 # src-ofi
 #
@@ -206,6 +224,8 @@ testing_perf_strided_SOURCES = testing/perf_strided.c
 testing_shift_SOURCES        = testing/shift.c
 testing_test_SOURCES         = testing/test.c
 
+
+
 ##############################################################################
 # the end
 #

comex/configure.ac

@@ -10,7 +10,7 @@ AC_INIT([Communication Runtime for Extreme Scale (comex)],
         [comex],
         [https://hpc.pnl.gov/globalarrays/])
 AC_CONFIG_AUX_DIR([build-aux])
-AC_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([src-common/comex.h])
 
@@ -49,10 +49,8 @@ COMEX_ENABLE_PROFILING_ARMCI
 AC_MSG_NOTICE
 AC_MSG_NOTICE([C compiler])
 AC_MSG_NOTICE
-
 COMEX_PROG_MPICC
 AC_USE_SYSTEM_EXTENSIONS
-
 AS_IF([test x$with_mpi_wrappers = xyes],
     [COMEX_MPI_UNWRAP],
     [COMEX_ARG_PARSE([with_mpi], [MPI_LIBS], [MPI_LDFLAGS], [MPI_CPPFLAGS])])
@@ -62,14 +60,21 @@ AS_CASE([$enable_mpi_tests],
 
 AM_CONDITIONAL([CROSS_COMPILING], [test "x$cross_compiling" = xyes])
 
-# Establish the underlying network infrastructure (MPI, OFA, etc)
+# Establish the underlying network infrastructure (MPI, OFA, OpenSHMEM, etc)
 COMEX_NETWORK_SETUP
 
+# If OpenSHMEM was selected, switch to using the OpenSHMEM compiler wrappers
+AS_IF([test "x$with_oshmem" = xyes],
+    [AC_MSG_NOTICE([Switching to OpenSHMEM compiler wrappers])
+     AS_IF([test "x$OSHCC" != x], [CC="$OSHCC"])
+     AS_IF([test "x$OSHCXX" != x], [CXX="$OSHCXX"])
+     AS_IF([test "x$OSHFC" != x], [FC="$OSHFC"; F77="$OSHFC"])
+     AC_MSG_NOTICE([Using CC=$CC, CXX=$CXX, FC=$FC])])
+
 # Checks for C header files.
 AC_HEADER_ASSERT
 AC_HEADER_DIRENT
 AC_HEADER_STDBOOL
-AC_HEADER_STDC
 AC_HEADER_SYS_WAIT
 COMEX_CHECK_HEADERS([assert.h])
 COMEX_CHECK_HEADERS([errno.h])

comex/m4/comex_network_setup.m4

@@ -208,6 +208,27 @@ AS_IF([test "x$happy" = xyes],
     [$2])
 ])dnl
 
+# _COMEX_NETWORK_OSHMEM([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+# ------------------------------------------------------------------
+AC_DEFUN([_COMEX_NETWORK_OSHMEM], [
+AC_MSG_NOTICE([searching for OpenSHMEM...])
+happy=yes
+# Check for OpenSHMEM compiler wrappers  
+AC_PATH_PROGS([OSHCC], [oshcc], [], [$PATH])
+AC_PATH_PROGS([OSHCXX], [oshcxx oshc++ oshCC], [], [$PATH])
+AC_PATH_PROGS([OSHFC], [oshfort oshfc], [], [$PATH])
+AS_IF([test "x$OSHCC" = x], [happy=no])
+AS_IF([test "x$happy" = xyes],
+    [# Found OpenSHMEM compiler wrappers
+     AC_MSG_NOTICE([found oshcc: $OSHCC])
+     AC_SUBST([OSHCC])
+     AC_SUBST([OSHCXX])
+     AC_SUBST([OSHFC])
+     comex_network=OSHMEM; with_oshmem=yes; $1],
+    [AC_MSG_NOTICE([oshcc not found])
+     $2])
+])dnl
+
 # COMEX_NETWORK_SETUP
 # -------------------
 # This macro allows user to choose the comex network but also allows the
@@ -231,6 +252,7 @@ _COMEX_NETWORK_WITH([mpi3],      [MPI-3 one-sided])
 _COMEX_NETWORK_WITH([ofa],       [Infiniband OpenIB])
 _COMEX_NETWORK_WITH([portals4],  [Portals4])
 _COMEX_NETWORK_WITH([ofi],       [OFI])
+_COMEX_NETWORK_WITH([oshmem],    [OpenSHMEM])
 # Temporarily add COMEX_NETWORK_CPPFLAGS to CPPFLAGS.
 comex_save_CPPFLAGS="$CPPFLAGS"; CPPFLAGS="$CPPFLAGS $COMEX_NETWORK_CPPFLAGS"
 # Temporarily add COMEX_NETWORK_LDFLAGS to LDFLAGS.
@@ -279,6 +301,9 @@ AS_IF([test "x$enable_autodetect" = xyes],
               AS_IF([test "x$comex_network" = xOFI],
                  [_COMEX_NETWORK_OFI([],
                     [AC_MSG_ERROR([test for COMEX_NETWORK=OFI failed])])])
+              AS_IF([test "x$comex_network" = xOSHMEM],
+                 [_COMEX_NETWORK_OSHMEM([],
+                    [AC_MSG_ERROR([test for COMEX_NETWORK=OSHMEM failed])])])
              ],
         [AC_MSG_WARN([too many comex networks specified: $comex_network_count])
          AC_MSG_WARN([the following were specified:])
@@ -290,6 +315,7 @@ AS_IF([test "x$enable_autodetect" = xyes],
          _COMEX_NETWORK_WARN([ofa])
          _COMEX_NETWORK_WARN([portals4])
          _COMEX_NETWORK_WARN([ofi])
+         _COMEX_NETWORK_WARN([oshmem])
          AC_MSG_ERROR([please select only one comex network])])])
 # Remove COMEX_NETWORK_CPPFLAGS from CPPFLAGS.
 CPPFLAGS="$comex_save_CPPFLAGS"
@@ -305,6 +331,7 @@ _COMEX_NETWORK_AM_CONDITIONAL([mpi3])
 _COMEX_NETWORK_AM_CONDITIONAL([ofa])
 _COMEX_NETWORK_AM_CONDITIONAL([portals4])
 _COMEX_NETWORK_AM_CONDITIONAL([ofi])
+_COMEX_NETWORK_AM_CONDITIONAL([oshmem])
 _COMEX_NETWORK_AC_DEFINE([mpi-ts])
 _COMEX_NETWORK_AC_DEFINE([mpi-mt])
 _COMEX_NETWORK_AC_DEFINE([mpi-pt])
@@ -313,6 +340,7 @@ _COMEX_NETWORK_AC_DEFINE([mpi3])
 _COMEX_NETWORK_AC_DEFINE([ofa])
 _COMEX_NETWORK_AC_DEFINE([portals4])
 _COMEX_NETWORK_AC_DEFINE([ofi])
+_COMEX_NETWORK_AC_DEFINE([oshmem])
 AC_SUBST([COMEX_NETWORK_LDFLAGS])
 AC_SUBST([COMEX_NETWORK_LIBS])
 AC_SUBST([COMEX_NETWORK_CPPFLAGS])