Online distributed graph partitioning using PT-Scotch

Makefile

@@ -759,6 +759,16 @@ endif
 	LIBS += $(NCLIB)
 endif
 
+ifneq "$(SCOTCH)" ""
+	SCOTCH_FCINCLUDES += -I$(SCOTCH)/include
+	SCOTCH_LIBS += -L$(SCOTCH)/lib64 -lptscotch -lscotch  -lptscotcherr -lm
+	SCOTCH_FFLAGS = -DMPAS_SCOTCH
+
+	FCINCLUDES += $(SCOTCH_FCINCLUDES)
+	CPPINCLUDES += $(SCOTCH_FCINCLUDES)
+	LIBS += $(SCOTCH_LIBS)
+	override CPPFLAGS += $(SCOTCH_FFLAGS)
+endif
 
 ifneq "$(PNETCDF)" ""
 ifneq ($(wildcard $(PNETCDF)/lib/libpnetcdf.*), )
@@ -1415,6 +1425,60 @@ musica_fortran_test:
 	$(eval MUSICA_FORTRAN_VERSION := $(shell pkg-config --modversion musica-fortran))
 	$(if $(findstring 1,$(MUSICA_FORTRAN_TEST)), $(info Built a simple test program with MUSICA-Fortran version $(MUSICA_FORTRAN_VERSION)), )
 
+scotch_fortran_test:
+	@#
+	@# Create C and Fortran test programs and try to build against the PT-SCOTCH library
+	@#
+	$(info Checking for a working Scotch library...)
+	$(eval SCOTCH_C_TEST := $(shell $\
+	    printf "#include <stdio.h>\n\
+			&#include \"mpi.h\"\n\
+			&#include \"ptscotch.h\"\n\
+			&int main(){\n\
+			&    int err;\n\
+			&    SCOTCH_Dgraph *dgraph;\n\
+			&    err = SCOTCH_dgraphInit(dgraph, MPI_COMM_WORLD);\n\
+			&    SCOTCH_dgraphExit(dgraph);\n\
+			&    return err;\n\
+			&}\n" | sed 's/&/ /' > ptscotch_c_test.c; $\
+		$\
+		$(CC) $(CPPINCLUDES) $(CFLAGS) $(LDFLAGS) ptscotch_c_test.c -o ptscotch_c_test.x $(SCOTCH_LIBS) > ptscotch_c_test.log 2>&1; $\
+		scotch_c_status=$$?; $\
+		if [ $$scotch_c_status -eq 0 ]; then $\
+			printf "1"; $\
+			rm -f ptscotch_c_test.c ptscotch_c_test.x ptscotch_c_test.log; $\
+		else $\
+			printf "0"; $\
+		fi $\
+	))
+	$(if $(findstring 0,$(SCOTCH_C_TEST)), $(error Could not build a simple C program with Scotch. $\
+		Test program ptscotch_c_test.c and output ptscotch_c_test.log have been left $\
+	    in the top-level MPAS directory for further debugging ))
+	$(if $(findstring 1,$(SCOTCH_C_TEST)), $(info Built a simple C program with Scotch ))
+	$(eval SCOTCH_FORTRAN_TEST := $(shell $\
+		printf "program test_scotch_fortran\n$\
+		&   include \"ptscotchf.h\"\n$\
+    	&   doubleprecision :: scotchgraph (scotch_graphdim)\n$\
+		&   integer :: ierr\n$\
+		&   ierr = 0\n$\
+		&   call scotchfgraphinit(scotchgraph (1), ierr)\n$\
+		&   call scotchfgraphexit(scotchgraph(1))\n$\
+		end program test_scotch_fortran\n" | sed 's/&/ /' > ptscotch_f_test.f90; $\
+		$\
+		$(FC) $(SCOTCH_FCINCLUDES) $(SCOTCH_FFLAGS) ptscotch_f_test.f90 -o ptscotch_f_test.x $(SCOTCH_LIBS) > ptscotch_f_test.log 2>&1; $\
+		scotch_fortran_status=$$?; $\
+		if [ $$scotch_fortran_status -eq 0 ]; then $\
+			printf "1"; $\
+			rm -f ptscotch_f_test.f90 ptscotch_f_test.x ptscotch_f_test.log; $\
+		else $\
+			printf "0"; $\
+		fi $\
+	))
+	$(if $(findstring 0,$(SCOTCH_FORTRAN_TEST)), $(error Could not build a simple Fortran program with Scotch. $\
+		Test program ptscotch_f_test.f90 and output ptscotch_f_test.log have been left $\
+	    in the top-level MPAS directory for further debugging ))
+	$(if $(findstring 1,$(SCOTCH_FORTRAN_TEST)), $(info Built a simple Fortran program with Scotch ))
+
 pnetcdf_test:
 	@#
 	@# Create test C programs that look for PNetCDF header file and some symbols in it
@@ -1471,6 +1535,13 @@ else
 MUSICA_MESSAGE = "MPAS was not linked with the MUSICA-Fortran library."
 endif
 
+ifneq "$(SCOTCH_FFLAGS)" ""
+MAIN_DEPS += scotch_fortran_test
+SCOTCH_MESSAGE = "MPAS has been linked with the Scotch graph partitioning library."
+else
+SCOTCH_MESSAGE = "MPAS was NOT linked with the Scotch graph partitioning library."
+endif
+
 mpas_main: $(MAIN_DEPS)
 	cd src; $(MAKE) FC="$(FC)" \
                  CC="$(CC)" \
@@ -1508,6 +1579,7 @@ mpas_main: $(MAIN_DEPS)
 	@echo $(OPENMP_OFFLOAD_MESSAGE)
 	@echo $(OPENACC_MESSAGE)
 	@echo $(MUSICA_MESSAGE)
+	@echo $(SCOTCH_MESSAGE)
 	@echo $(SHAREDLIB_MESSAGE)
 ifeq "$(AUTOCLEAN)" "true"
 	@echo $(AUTOCLEAN_MESSAGE)

src/framework/Makefile

@@ -36,6 +36,8 @@ OBJS = mpas_kind_types.o \
        mpas_log.o \
        mpas_halo.o \
        mpas_string_utils.o \
+       mpas_ptscotch_interface.o \
+       ptscotch_interface.o \
        mpas_stream_inquiry.o \
        stream_inquiry.o
 
@@ -85,7 +87,7 @@ mpas_timekeeping.o: mpas_string_utils.o mpas_kind_types.o mpas_derived_types.o m
 
 mpas_timer.o: mpas_kind_types.o mpas_dmpar.o mpas_threading.o mpas_log.o
 
-mpas_block_decomp.o: mpas_derived_types.o mpas_hash.o mpas_io_units.o mpas_dmpar.o
+mpas_block_decomp.o: mpas_derived_types.o mpas_hash.o mpas_io_units.o mpas_dmpar.o mpas_ptscotch_interface.o
 
 mpas_block_creator.o: mpas_dmpar.o mpas_hash.o mpas_sort.o mpas_io_units.o mpas_block_decomp.o mpas_stream_manager.o mpas_decomp.o mpas_abort.o $(DEPS)
 
@@ -112,6 +114,8 @@ xml_stream_parser.o: xml_stream_parser.c
 
 mpas_halo.o: mpas_derived_types.o mpas_pool_routines.o mpas_log.o
 
+mpas_ptscotch_interface.o : mpas_derived_types.o mpas_dmpar.o mpas_log.o ptscotch_interface.o
+
 mpas_stream_inquiry.o : mpas_derived_types.o mpas_log.o mpas_c_interfacing.o
 
 clean: