From 1d7ab067d57773111206d7895e6ee12d614d1ded Mon Sep 17 00:00:00 2001
From: Ben Dudson <dudson2@llnl.gov>
Date: Fri, 9 May 2025 16:46:51 -0700
Subject: [PATCH] periodicX communication fixes

- Fix mesh::(send/recv)X(In/Out) so that it works with periodicX=true

- FV::communicateFluxes now communicates when periodicX=true
---
 src/mesh/fv_ops.cxx              |  5 ++-
 src/mesh/impls/bout/boutmesh.cxx | 64 ++++++++++++++++++++++++--------
 2 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/src/mesh/fv_ops.cxx b/src/mesh/fv_ops.cxx
index cd5b924e9e..fe5422b4d1 100644
--- a/src/mesh/fv_ops.cxx
+++ b/src/mesh/fv_ops.cxx
@@ -442,8 +442,9 @@ void communicateFluxes(Field3D& f) {
   comm_handle xin, xout;
   // Cache results to silence spurious compiler warning about xin,
   // xout possibly being uninitialised when used
-  bool not_first = !mesh->firstX();
-  bool not_last = !mesh->lastX();
+  const bool not_first = mesh->periodicX || !mesh->firstX();
+  const bool not_last = mesh->periodicX || !mesh->lastX();
+
   if (not_first) {
     xin = mesh->irecvXIn(f(0, 0), size, 0);
   }
diff --git a/src/mesh/impls/bout/boutmesh.cxx b/src/mesh/impls/bout/boutmesh.cxx
index 16061cd47e..31e9c6112d 100644
--- a/src/mesh/impls/bout/boutmesh.cxx
+++ b/src/mesh/impls/bout/boutmesh.cxx
@@ -1534,42 +1534,66 @@ bool BoutMesh::firstX() const { return PE_XIND == 0; }
 bool BoutMesh::lastX() const { return PE_XIND == NXPE - 1; }
 
 int BoutMesh::sendXOut(BoutReal* buffer, int size, int tag) {
+  Timer timer("comms");
+
+  int proc {-1};
   if (PE_XIND == NXPE - 1) {
-    return 1;
+    if (periodicX) {
+      // Wrap around to first processor in X
+      proc = PROC_NUM(0, PE_YIND);
+    } else {
+      return 1;
+    }
+  } else {
+    proc = PROC_NUM(PE_XIND + 1, PE_YIND);
   }
 
-  Timer timer("comms");
-
-  mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, PROC_NUM(PE_XIND + 1, PE_YIND), tag,
+  mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag,
                 BoutComm::get());
 
   return 0;
 }
 
 int BoutMesh::sendXIn(BoutReal* buffer, int size, int tag) {
+  Timer timer("comms");
+
+  int proc {-1};
   if (PE_XIND == 0) {
-    return 1;
+    if (periodicX) {
+      // Wrap around to last processor in X
+      proc = PROC_NUM(NXPE - 1, PE_YIND);
+    } else {
+      return 1;
+    }
+  } else {
+    proc = PROC_NUM(PE_XIND - 1, PE_YIND);
   }
 
-  Timer timer("comms");
-
-  mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, PROC_NUM(PE_XIND - 1, PE_YIND), tag,
+  mpi->MPI_Send(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag,
                 BoutComm::get());
 
   return 0;
 }
 
 comm_handle BoutMesh::irecvXOut(BoutReal* buffer, int size, int tag) {
+  Timer timer("comms");
+
+  int proc {-1};
   if (PE_XIND == NXPE - 1) {
-    return nullptr;
+    if (periodicX) {
+      // Wrap around to first processor in X
+      proc = PROC_NUM(0, PE_YIND);
+    } else {
+      return nullptr;
+    }
+  } else {
+    proc = PROC_NUM(PE_XIND + 1, PE_YIND);
   }
 
-  Timer timer("comms");
-
   // Get a communications handle. Not fussy about size of arrays
   CommHandle* ch = get_handle(0, 0);
 
-  mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, PROC_NUM(PE_XIND + 1, PE_YIND), tag,
+  mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag,
                  BoutComm::get(), ch->request);
 
   ch->in_progress = true;
@@ -1578,16 +1602,24 @@ comm_handle BoutMesh::irecvXOut(BoutReal* buffer, int size, int tag) {
 }
 
 comm_handle BoutMesh::irecvXIn(BoutReal* buffer, int size, int tag) {
+  Timer timer("comms");
+
+  int proc {-1};
   if (PE_XIND == 0) {
-    return nullptr;
+    if (periodicX) {
+      // Wrap around to last processor in X
+      proc = PROC_NUM(NXPE - 1, PE_YIND);
+    } else {
+      return nullptr;
+    }
+  } else {
+    proc = PROC_NUM(PE_XIND - 1, PE_YIND);
   }
 
-  Timer timer("comms");
-
   // Get a communications handle. Not fussy about size of arrays
   CommHandle* ch = get_handle(0, 0);
 
-  mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, PROC_NUM(PE_XIND - 1, PE_YIND), tag,
+  mpi->MPI_Irecv(buffer, size, PVEC_REAL_MPI_TYPE, proc, tag,
                  BoutComm::get(), ch->request);
 
   ch->in_progress = true;