Allow distributed partition boundaries to be not just rank boundaries

doc/conf.py

@@ -38,6 +38,7 @@
     "https://tisaac.gitlab.io/recursivenodes/": None,
     "https://fenics.readthedocs.io/projects/fiat/en/latest/": None,
     "https://finat.github.io/FInAT/": None,
+    "https://mpi4py.readthedocs.io/en/stable": None,
     "h5py": ("https://docs.h5py.org/en/stable", None),
 }
 

meshmode/discretization/connection/direct.py

@@ -20,7 +20,6 @@
 THE SOFTWARE.
 """
 
-
 import numpy as np
 import numpy.linalg as la
 from abc import ABC, abstractmethod
@@ -244,6 +243,8 @@ class DiscretizationConnectionElementGroup:
 
         A list of :class:`InterpolationBatch` instances.
     """
+    batches: Sequence[InterpolationBatch]
+
     def __init__(self, batches):
         self.batches = batches
 
@@ -341,6 +342,7 @@ class DirectDiscretizationConnection(DiscretizationConnection):
 
     .. automethod:: __call__
     """
+    groups: Sequence[DiscretizationConnectionElementGroup]
 
     def __init__(self,
             from_discr: Discretization, to_discr: Discretization,

meshmode/discretization/connection/opposite_face.py

@@ -521,21 +521,20 @@ def make_opposite_face_connection(actx, volume_to_bdry_conn):
 
 # {{{ make_partition_connection
 
-def make_partition_connection(actx, *, local_bdry_conn, i_local_part,
+# FIXME: Consider adjusting terminology from local/remote to self/other.
+def make_partition_connection(actx, *, local_bdry_conn,
         remote_bdry_discr, remote_group_infos):
     """
-    Connects ``local_bdry_conn`` to a neighboring partition.
+    Connects ``local_bdry_conn`` to a neighboring part.
 
-    :arg local_bdry_conn: A :class:`DiscretizationConnection` of the local
-        partition.
-    :arg i_local_part: The partition number of the local partition.
+    :arg local_bdry_conn: A :class:`DiscretizationConnection` of the local part.
     :arg remote_bdry_discr: A :class:`~meshmode.discretization.Discretization`
-        of the boundary of the remote partition.
+        of the boundary of the remote part.
     :arg remote_group_infos: An array of
         :class:`meshmode.distributed.RemoteGroupInfo` instances, one per remote
         volume element group.
     :returns: A :class:`DirectDiscretizationConnection` that performs data
-        exchange across faces from the remote partition to partition `i_local_part`.
+        exchange across faces from the remote part to the local part.
 
     .. versionadded:: 2017.1
 
@@ -557,15 +556,15 @@ def make_partition_connection(actx, *, local_bdry_conn, i_local_part,
     # The code assumes that there is the same number of volume and surface groups.
     #
     # A weak reason to choose remote as the outer loop is because
-    # InterPartitionAdjacency refers to neighbors by global volume element
+    # InterPartAdjacency refers to neighbors by global volume element
     # numbers, and we only have enough information to resolve those to (group,
     # group_local_el_nr) for local elements (whereas we have no information
     # about remote volume elements).
     #
     # (See the find_group_indices below.)
 
     for rgi in remote_group_infos:
-        rem_ipags = rgi.inter_partition_adj_groups
+        rem_ipags = rgi.inter_part_adj_groups
 
         for rem_ipag in rem_ipags:
             i_local_grps = find_group_indices(local_vol_groups, rem_ipag.neighbors)