From 77c8946daf739fff46387622cdb1ddcedb832803 Mon Sep 17 00:00:00 2001
From: David Bold <dave@ipp.mpg.de>
Date: Mon, 13 Nov 2023 11:17:19 +0100
Subject: [PATCH 01/17] Make squashing faster

---
 src/boutdata/squashoutput.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/boutdata/squashoutput.py b/src/boutdata/squashoutput.py
index e32a7e43d..0d417712a 100644
--- a/src/boutdata/squashoutput.py
+++ b/src/boutdata/squashoutput.py
@@ -240,7 +240,7 @@ def squashoutput(
                 f.write(varname, var)
 
         var = None
-        gc.collect()
+        # gc.collect()
 
     # Copy file attributes
     for attrname in outputs.list_file_attributes():
@@ -254,7 +254,6 @@ def squashoutput(
         f.close()
 
     del outputs
-    gc.collect()
 
     if delete:
         if append:

From 2e48c55de35ae5c8e80622eb0715bf454199f559 Mon Sep 17 00:00:00 2001
From: David Bold <dave@ipp.mpg.de>
Date: Mon, 13 Nov 2023 12:38:08 +0100
Subject: [PATCH 02/17] Reduce amount of combinations we test

---
 src/boutdata/tests/test_collect.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index e34931466..2ece8ea28 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -23,6 +23,11 @@
 # Note - using tmp_path fixture requires pytest>=3.9.0
 
 collect_kwargs_list = [
+    {"xguards": True, "yguards": "include_upper"},
+    {"xguards": False, "yguards": False},
+]
+
+collect_kwargs_list_full = [
     {"xguards": True, "yguards": "include_upper"},
     {"xguards": False, "yguards": "include_upper"},
     {"xguards": True, "yguards": True},
@@ -31,10 +36,10 @@
     {"xguards": False, "yguards": False},
 ]
 
+
 squash_params_list = [
     (False, {}),
     (True, {}),
-    (True, {"parallel": 2}),
 ]
 
 
@@ -290,12 +295,8 @@ def test_core_min_files_existing_squash_file_raises(self, tmp_path, time_split):
         "time_split",
         [
             (1, None),
-            (2, None),
             (2, 3),
-            (3, None),
-            (4, None),
             (5, None),
-            (6, None),
             (7, None),
         ],
     )
@@ -874,11 +875,6 @@ def test_singlenull(self, tmp_path, squash_params, collect_kwargs):
             # {"parallel": False},
             {"parallel": 1},
             {"parallel": 2},
-            {"parallel": 3},
-            {"parallel": 4},
-            {"parallel": 5},
-            {"parallel": 6},
-            {"parallel": 7},
             {"parallel": 8},
             {"parallel": True},
         ),
@@ -1396,7 +1392,7 @@ def test_singlenull_tind_xind_yind_zind(
         )
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
+    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list_full)
     def test_connected_doublenull_min_files(
         self, tmp_path, squash_params, collect_kwargs
     ):
@@ -1736,13 +1732,6 @@ def test_disconnected_doublenull(
         [
             {},
             {"compress": True, "complevel": 1},
-            {"compress": True, "complevel": 2},
-            {"compress": True, "complevel": 3},
-            {"compress": True, "complevel": 4},
-            {"compress": True, "complevel": 5},
-            {"compress": True, "complevel": 5},
-            {"compress": True, "complevel": 7},
-            {"compress": True, "complevel": 8},
             {"compress": True, "complevel": 9},
         ],
     )

From ed7c21361bf8c91b4ec4af7e0587257a1f0f9887 Mon Sep 17 00:00:00 2001
From: David Bold <dave@ipp.mpg.de>
Date: Mon, 13 Nov 2023 13:41:13 +0100
Subject: [PATCH 03/17] Do not force --cov for pytest

---
 pyproject.toml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 6259b77d6..2f8353000 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,7 +43,6 @@ dynamic = ["version"]
 [project.optional-dependencies]
 tests = [
     "pytest",
-    "pytest-cov",
 ]
 docs = [
     "sphinx>=3.4,<5",
@@ -57,6 +56,3 @@ version = { attr = "setuptools_scm.get_version" }
 
 [tool.setuptools_scm]
 write_to = "src/boutdata/_version.py"
-
-[tool.pytest.ini_options]
-addopts = "--cov=boutdata"

From 50477463c8616b7976fba4df0e448ecdeeee7f59 Mon Sep 17 00:00:00 2001
From: David Bold <dave@ipp.mpg.de>
Date: Mon, 13 Nov 2023 13:46:47 +0100
Subject: [PATCH 04/17] Remove last reference of gc

---
 src/boutdata/squashoutput.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/boutdata/squashoutput.py b/src/boutdata/squashoutput.py
index 0d417712a..20ff90503 100644
--- a/src/boutdata/squashoutput.py
+++ b/src/boutdata/squashoutput.py
@@ -110,7 +110,6 @@ def squashoutput(
         is used.
     """
     # use local imports to allow fast import for tab-completion
-    import gc
     import glob
     import os
     import shutil
@@ -240,7 +239,6 @@ def squashoutput(
                 f.write(varname, var)
 
         var = None
-        # gc.collect()
 
     # Copy file attributes
     for attrname in outputs.list_file_attributes():

From cdeb1d2e2b0a298129a584edefea603460c1b465 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 24 Jul 2024 15:28:58 +0100
Subject: [PATCH 05/17] Restore parallel squash collect tests

---
 src/boutdata/tests/test_collect.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index 2ece8ea28..a7a1a3790 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -40,6 +40,7 @@
 squash_params_list = [
     (False, {}),
     (True, {}),
+    (True, {"parallel": 2}),
 ]
 
 

From aa67659f606c502a5a023ac5bbacc970b8c9298f Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 31 Jul 2024 11:12:18 +0100
Subject: [PATCH 06/17] Cache dump file creation for collect tests

---
 src/boutdata/tests/test_collect.py | 1409 ++++++++++++----------------
 1 file changed, 616 insertions(+), 793 deletions(-)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index a7a1a3790..dbc590a14 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -1,3 +1,4 @@
+import copy
 from glob import glob
 from pathlib import Path
 
@@ -45,7 +46,7 @@
 
 
 def check_collected_data(
-    expected,
+    expected_in,
     *,
     fieldperp_global_yind,
     doublenull,
@@ -75,6 +76,8 @@ def check_collected_data(
     squash_kwargs : dict, optional
         Keyword arguments passed to `squashoutput()`.
     """
+    expected = copy.deepcopy(expected_in)
+
     # Apply effect of arguments to expected data
     if not collect_kwargs["xguards"]:
         remove_xboundaries(expected, expected["MXG"])
@@ -121,6 +124,518 @@ def check_collected_data(
             assert f.read_file_attribute(attrname) == attr
 
 
+def symlink_dump_files(src: Path, dst: Path):
+    """Symlink all dump files from ``src`` directory into ``dst``"""
+    for f in src.glob("*.nc"):
+        (dst / f.name).symlink_to(f)
+
+
+def create_dump_file_set(
+    grid_info, fieldperp_global_yind, tmp_path, rng, dump_params, fieldperp_yproc_ind=0
+):
+    """Create a set of dump files based on ``dump_params`` and return the concatenated data"""
+
+    dumps = []
+    for i, boundaries, fieldperp_yind in dump_params:
+        dumps.append(
+            create_dump_file(
+                tmpdir=tmp_path,
+                rng=rng,
+                grid_info=grid_info,
+                i=i,
+                boundaries=boundaries,
+                fieldperp_global_yind=fieldperp_yind,
+            )
+        )
+
+    expected = concatenate_data(
+        dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
+    )
+    return expected
+
+
+@pytest.fixture(scope="module")
+def core_min(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "core_min"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 3
+
+    # core
+    # core includes "ylower" and "yupper" even though there is no actual y-boundary
+    # because collect/squashoutput collect these points
+    dump_params = [
+        (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(100),
+        dump_params,
+    )
+
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def core_full(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "core_full"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 3
+
+    # core
+    # core includes "ylower" and "yupper" even though there is no actual y-boundary
+    # because collect/squashoutput collect these points
+    dump_params = [
+        (0, ["xinner", "ylower"], fieldperp_global_yind),
+        (1, ["ylower"], fieldperp_global_yind),
+        (2, ["xouter", "ylower"], fieldperp_global_yind),
+        (3, ["xinner"], -1),
+        (4, [], -1),
+        (5, ["xouter"], -1),
+        (6, ["xinner", "yupper"], -1),
+        (7, ["yupper"], -1),
+        (8, ["xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nxpe=3, nype=3),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(101),
+        dump_params,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def sol_min(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "sol_min"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 3
+
+    # SOL
+    dump_params = [
+        (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(ixseps1=0, ixseps2=0),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(102),
+        dump_params,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def sol_full(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "sol_full"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 3
+
+    # SOL
+    dump_params = [
+        (0, ["xinner", "ylower"], fieldperp_global_yind),
+        (1, ["ylower"], fieldperp_global_yind),
+        (2, ["xouter", "ylower"], fieldperp_global_yind),
+        (3, ["xinner"], -1),
+        (4, [], -1),
+        (5, ["xouter"], -1),
+        (6, ["xinner", "yupper"], -1),
+        (7, ["yupper"], -1),
+        (8, ["xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nxpe=3, nype=3, ixseps1=0, ixseps2=0),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(103),
+        dump_params,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def single_null_min(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "single_null_min"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 7
+
+    dump_params = [
+        # inner divertor leg
+        (0, ["xinner", "xouter", "ylower"], -1),
+        # core
+        (1, ["xinner", "xouter"], fieldperp_global_yind),
+        # outer divertor leg
+        (2, ["xinner", "xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nype=3, ixseps1=4, xpoints=1),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(104),
+        dump_params,
+        fieldperp_yproc_ind=1,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def single_null_lower_boundary(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "single_null_lower_boundary"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 1
+
+    dump_params = [
+        # inner divertor leg
+        (0, ["xinner", "xouter", "ylower"], fieldperp_global_yind),
+        # core
+        (1, ["xinner", "xouter"], -1),
+        # outer divertor leg
+        (2, ["xinner", "xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nype=3, ixseps1=4, xpoints=1),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(104),
+        dump_params,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def single_null_upper_boundary(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "single_null_upper_boundary"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 14
+
+    dump_params = [
+        # inner divertor leg
+        (0, ["xinner", "xouter", "ylower"], -1),
+        # core
+        (1, ["xinner", "xouter"], -1),
+        # outer divertor leg
+        (2, ["xinner", "xouter", "yupper"], fieldperp_global_yind),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nype=3, ixseps1=4, xpoints=1),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(104),
+        dump_params,
+        fieldperp_yproc_ind=2,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def single_null_inconsistent(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "single_null_inconsistent"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 7
+
+    dump_params = [
+        # inner divertor leg
+        (0, ["xinner", "xouter", "ylower"], 2),
+        # core
+        (1, ["xinner", "xouter"], 7),
+        # outer divertor leg
+        (2, ["xinner", "xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nype=3, ixseps1=4, xpoints=1),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(104),
+        dump_params,
+        fieldperp_yproc_ind=1,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def single_null_full(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "single_null_full"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 19
+
+    dump_params = [
+        # inner divertor leg
+        (0, ["xinner", "ylower"], -1),
+        (1, ["ylower"], -1),
+        (2, ["xouter", "ylower"], -1),
+        (3, ["xinner"], -1),
+        (4, [], -1),
+        (5, ["xouter"], -1),
+        (6, ["xinner"], -1),
+        (7, [], -1),
+        (8, ["xouter"], -1),
+        # core
+        (9, ["xinner"], -1),
+        (10, [], -1),
+        (11, ["xouter"], -1),
+        (12, ["xinner"], fieldperp_global_yind),
+        (13, [], fieldperp_global_yind),
+        (14, ["xouter"], fieldperp_global_yind),
+        (15, ["xinner"], -1),
+        (16, [], -1),
+        (17, ["xouter"], -1),
+        # outer divertor leg
+        (18, ["xinner"], -1),
+        (19, [], -1),
+        (20, ["xouter"], -1),
+        (21, ["xinner"], -1),
+        (22, [], -1),
+        (23, ["xouter"], -1),
+        (24, ["xinner", "yupper"], -1),
+        (25, ["yupper"], -1),
+        (26, ["xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nxpe=3, nype=9, ixseps1=7, xpoints=1),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(105),
+        dump_params,
+        fieldperp_yproc_ind=4,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def connected_double_null_min(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "connected_double_null_min"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 7
+
+    dump_params = [
+        # inner, lower divertor leg
+        (0, ["xinner", "xouter", "ylower"], -1),
+        # inner core
+        (1, ["xinner", "xouter"], fieldperp_global_yind),
+        # inner, upper divertor leg
+        (2, ["xinner", "xouter", "yupper"], -1),
+        # outer, upper divertor leg
+        (3, ["xinner", "xouter", "ylower"], -1),
+        # outer core
+        (4, ["xinner", "xouter"], -1),
+        # outer, lower divertor leg
+        (5, ["xinner", "xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nype=6, ixseps1=4, ixseps2=4, xpoints=2),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(107),
+        dump_params,
+        fieldperp_yproc_ind=1,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def connected_double_null_full(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "connected_double_null_full"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 19
+
+    dump_params = [
+        # inner, lower divertor leg
+        (0, ["xinner", "ylower"], -1),
+        (1, ["ylower"], -1),
+        (2, ["xouter", "ylower"], -1),
+        (3, ["xinner"], -1),
+        (4, [], -1),
+        (5, ["xouter"], -1),
+        (6, ["xinner"], -1),
+        (7, [], -1),
+        (8, ["xouter"], -1),
+        # inner core
+        (9, ["xinner"], -1),
+        (10, [], -1),
+        (11, ["xouter"], -1),
+        (12, ["xinner"], fieldperp_global_yind),
+        (13, [], fieldperp_global_yind),
+        (14, ["xouter"], fieldperp_global_yind),
+        (15, ["xinner"], -1),
+        (16, [], -1),
+        (17, ["xouter"], -1),
+        # inner, upper divertor leg
+        (18, ["xinner"], -1),
+        (19, [], -1),
+        (20, ["xouter"], -1),
+        (21, ["xinner"], -1),
+        (22, [], -1),
+        (23, ["xouter"], -1),
+        (24, ["xinner", "yupper"], -1),
+        (25, ["yupper"], -1),
+        (26, ["xouter", "yupper"], -1),
+        # outer, upper divertor leg
+        (27, ["xinner", "ylower"], -1),
+        (28, ["ylower"], -1),
+        (29, ["xouter", "ylower"], -1),
+        (30, ["xinner"], -1),
+        (31, [], -1),
+        (32, ["xouter"], -1),
+        (33, ["xinner"], -1),
+        (34, [], -1),
+        (35, ["xouter"], -1),
+        # outer core
+        (36, ["xinner"], -1),
+        (37, [], -1),
+        (38, ["xouter"], -1),
+        (39, ["xinner"], -1),
+        (40, [], -1),
+        (41, ["xouter"], -1),
+        (42, ["xinner"], -1),
+        (43, [], -1),
+        (44, ["xouter"], -1),
+        # outer, lower divertor leg
+        (45, ["xinner"], -1),
+        (46, [], -1),
+        (47, ["xouter"], -1),
+        (48, ["xinner"], -1),
+        (49, [], -1),
+        (50, ["xouter"], -1),
+        (51, ["xinner", "yupper"], -1),
+        (52, ["yupper"], -1),
+        (53, ["xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nxpe=3, nype=18, ixseps1=7, ixseps2=7, xpoints=2),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(108),
+        dump_params,
+        fieldperp_yproc_ind=4,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def disconnected_double_null_min(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "disconnected_double_null_min"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 7
+
+    dump_params = [
+        # inner, lower divertor leg
+        (0, ["xinner", "xouter", "ylower"], -1),
+        # inner core
+        (1, ["xinner", "xouter"], fieldperp_global_yind),
+        # inner, upper divertor leg
+        (2, ["xinner", "xouter", "yupper"], -1),
+        # outer, upper divertor leg
+        (3, ["xinner", "xouter", "ylower"], -1),
+        # outer core
+        (4, ["xinner", "xouter"], -1),
+        # outer, lower divertor leg
+        (5, ["xinner", "xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nype=6, ixseps1=3, ixseps2=5, xpoints=2),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(109),
+        dump_params,
+        fieldperp_yproc_ind=1,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def disconnected_double_null_full(tmp_path_factory):
+    tmp_path = tmp_path_factory.getbasetemp() / "disconnected_double_null_full"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 19
+
+    dump_params = [
+        # inner, lower divertor leg
+        (0, ["xinner", "ylower"], -1),
+        (1, ["ylower"], -1),
+        (2, ["xouter", "ylower"], -1),
+        (3, ["xinner"], -1),
+        (4, [], -1),
+        (5, ["xouter"], -1),
+        (6, ["xinner"], -1),
+        (7, [], -1),
+        (8, ["xouter"], -1),
+        # inner core
+        (9, ["xinner"], -1),
+        (10, [], -1),
+        (11, ["xouter"], -1),
+        (12, ["xinner"], fieldperp_global_yind),
+        (13, [], fieldperp_global_yind),
+        (14, ["xouter"], fieldperp_global_yind),
+        (15, ["xinner"], -1),
+        (16, [], -1),
+        (17, ["xouter"], -1),
+        # inner, upper divertor leg
+        (18, ["xinner"], -1),
+        (19, [], -1),
+        (20, ["xouter"], -1),
+        (21, ["xinner"], -1),
+        (22, [], -1),
+        (23, ["xouter"], -1),
+        (24, ["xinner", "yupper"], -1),
+        (25, ["yupper"], -1),
+        (26, ["xouter", "yupper"], -1),
+        # outer, upper divertor leg
+        (27, ["xinner", "ylower"], -1),
+        (28, ["ylower"], -1),
+        (29, ["xouter", "ylower"], -1),
+        (30, ["xinner"], -1),
+        (31, [], -1),
+        (32, ["xouter"], -1),
+        (33, ["xinner"], -1),
+        (34, [], -1),
+        (35, ["xouter"], -1),
+        # outer core
+        (36, ["xinner"], -1),
+        (37, [], -1),
+        (38, ["xouter"], -1),
+        (39, ["xinner"], -1),
+        (40, [], -1),
+        (41, ["xouter"], -1),
+        (42, ["xinner"], -1),
+        (43, [], -1),
+        (44, ["xouter"], -1),
+        # outer, lower divertor leg
+        (45, ["xinner"], -1),
+        (46, [], -1),
+        (47, ["xouter"], -1),
+        (48, ["xinner"], -1),
+        (49, [], -1),
+        (50, ["xouter"], -1),
+        (51, ["xinner", "yupper"], -1),
+        (52, ["yupper"], -1),
+        (53, ["xouter", "yupper"], -1),
+    ]
+    expected = create_dump_file_set(
+        make_grid_info(nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(110),
+        dump_params,
+        fieldperp_yproc_ind=4,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
 def check_variable(
     varname, actual, expected_data, expected_attributes, fieldperp_global_yind
 ):
@@ -140,7 +655,7 @@ def check_variable(
     fieldperp_global_yind : int
         Global y-index where FieldPerps have been created
     """
-    npt.assert_array_equal(expected_data, actual)
+    npt.assert_array_equal(expected_data, actual, err_msg=varname)
     actual_keys = list(actual.attributes.keys())
     if expected_attributes is not None:
         for a in expected_attributes:
@@ -174,41 +689,13 @@ def check_variable(
 class TestCollect:
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_core_min_files(self, tmp_path, squash_params, collect_kwargs):
+    def test_core_min_files(self, core_min, tmp_path, squash_params, collect_kwargs):
         """
         Check output from a core-only case using the minimum number of processes
         """
         squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info()
-
-        fieldperp_global_yind = 3
-        fieldperp_yproc_ind = 0
-
-        rng = np.random.default_rng(100)
-
-        # core
-        # core includes "ylower" and "yupper" even though there is no actual y-boundary
-        # because collect/squashoutput collect these points
-        dump_params = [
-            (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        data_path, expected, fieldperp_global_yind = core_min
+        symlink_dump_files(data_path, tmp_path)
 
         check_collected_data(
             expected,
@@ -234,11 +721,15 @@ def test_core_min_files(self, tmp_path, squash_params, collect_kwargs):
             (7, None),
         ],
     )
-    def test_core_min_files_existing_squash_file_raises(self, tmp_path, time_split):
+    def test_core_min_files_existing_squash_file_raises(
+        self, core_min, tmp_path, time_split
+    ):
         """
         Check output from a core-only case using the minimum number of processes
         """
         time_split_size, time_split_first_label = time_split
+        data_path, _, _ = core_min
+        symlink_dump_files(data_path, tmp_path)
 
         squash_kwargs = {}
         if time_split_size is not None:
@@ -246,28 +737,6 @@ def test_core_min_files_existing_squash_file_raises(self, tmp_path, time_split):
         if time_split_first_label is not None:
             squash_kwargs["time_split_first_label"] = time_split_first_label
 
-        grid_info = make_grid_info()
-
-        fieldperp_global_yind = 3
-
-        rng = np.random.default_rng(100)
-
-        # core
-        # core includes "ylower" and "yupper" even though there is no actual y-boundary
-        # because collect/squashoutput collect these points
-        dump_params = [
-            (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
-        ]
-        for i, boundaries, fieldperp_yind in dump_params:
-            create_dump_file(
-                tmpdir=tmp_path,
-                rng=rng,
-                grid_info=grid_info,
-                i=i,
-                boundaries=boundaries,
-                fieldperp_global_yind=fieldperp_yind,
-            )
-
         if time_split_size is None:
             filenames = ["boutdata.nc"]
         else:
@@ -302,45 +771,18 @@ def test_core_min_files_existing_squash_file_raises(self, tmp_path, time_split):
         ],
     )
     @pytest.mark.parametrize("parallel", [False, 2])
-    def test_core_min_files_time_split(self, tmp_path, time_split, parallel):
+    def test_core_min_files_time_split(self, core_min, tmp_path, time_split, parallel):
         """
         Check output from a core-only case using the minimum number of processes
         """
+        data_path, expected, fieldperp_global_yind = core_min
+        symlink_dump_files(data_path, tmp_path)
+
         collect_kwargs = {"xguards": True, "yguards": "include_upper"}
         squash_kwargs = {"time_split_size": time_split[0], "parallel": parallel}
         if time_split[1] is not None:
             squash_kwargs["time_split_first_label"] = time_split[1]
 
-        grid_info = make_grid_info()
-
-        fieldperp_global_yind = 3
-        fieldperp_yproc_ind = 0
-
-        rng = np.random.default_rng(100)
-
-        # core
-        # core includes "ylower" and "yupper" even though there is no actual y-boundary
-        # because collect/squashoutput collect these points
-        dump_params = [
-            (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
-
         # Copy of check_collected_data code, modified to test series of output
         # files created when setting time_split_size
         ######################################################################
@@ -386,35 +828,16 @@ def test_core_min_files_time_split(self, tmp_path, time_split, parallel):
                     fieldperp_global_yind,
                 )
 
-    def test_core_min_files_append_time_split_raises(self, tmp_path):
+    def test_core_min_files_append_time_split_raises(self, core_min, tmp_path):
         """
         Check output from a core-only case using the minimum number of processes
         """
+        data_path, expected, fieldperp_global_yind = core_min
+        symlink_dump_files(data_path, tmp_path)
+
         collect_kwargs = {"xguards": True, "yguards": "include_upper"}
         squash_kwargs = {"time_split_size": 2, "append": True}
 
-        grid_info = make_grid_info()
-
-        fieldperp_global_yind = 3
-
-        rng = np.random.default_rng(100)
-
-        # core
-        # core includes "ylower" and "yupper" even though there is no actual y-boundary
-        # because collect/squashoutput collect these points
-        dump_params = [
-            (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
-        ]
-        for i, boundaries, fieldperp_yind in dump_params:
-            create_dump_file(
-                tmpdir=tmp_path,
-                rng=rng,
-                grid_info=grid_info,
-                i=i,
-                boundaries=boundaries,
-                fieldperp_global_yind=fieldperp_yind,
-            )
-
         with pytest.raises(
             ValueError, match="'time_split_size' is not compatible with append=True"
         ):
@@ -424,51 +847,16 @@ def test_core_min_files_append_time_split_raises(self, tmp_path):
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_core(self, tmp_path, squash_params, collect_kwargs):
+    def test_core(self, core_full, tmp_path, squash_params, collect_kwargs):
         """
         Check output from a core-only case using a large number of processes. 'Large'
         means there is at least one process in each region with no edges touching
         another region.
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(nxpe=3, nype=3)
-
-        fieldperp_global_yind = 3
-        fieldperp_yproc_ind = 0
-
-        rng = np.random.default_rng(101)
-
-        # core
-        # core includes "ylower" and "yupper" even though there is no actual y-boundary
-        # because collect/squashoutput collect these points
-        dump_params = [
-            (0, ["xinner", "ylower"], fieldperp_global_yind),
-            (1, ["ylower"], fieldperp_global_yind),
-            (2, ["xouter", "ylower"], fieldperp_global_yind),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner", "yupper"], -1),
-            (7, ["yupper"], -1),
-            (8, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
+        data_path, expected, fieldperp_global_yind = core_full
+        symlink_dump_files(data_path, tmp_path)
 
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -482,39 +870,14 @@ def test_core(self, tmp_path, squash_params, collect_kwargs):
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_sol_min_files(self, tmp_path, squash_params, collect_kwargs):
+    def test_sol_min_files(self, sol_min, tmp_path, squash_params, collect_kwargs):
         """
         Check output from a SOL-only case using the minimum number of processes
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(ixseps1=0, ixseps2=0)
-
-        fieldperp_global_yind = 3
-        fieldperp_yproc_ind = 0
-
-        rng = np.random.default_rng(102)
-
-        # SOL
-        dump_params = [
-            (0, ["xinner", "xouter", "ylower", "yupper"], fieldperp_global_yind),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
+        data_path, expected, fieldperp_global_yind = sol_min
+        symlink_dump_files(data_path, tmp_path)
 
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -528,49 +891,16 @@ def test_sol_min_files(self, tmp_path, squash_params, collect_kwargs):
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_sol(self, tmp_path, squash_params, collect_kwargs):
+    def test_sol(self, sol_full, tmp_path, squash_params, collect_kwargs):
         """
         Check output from a SOL-only case using a large number of processes. 'Large'
         means there is at least one process in each region with no edges touching
         another region.
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(nxpe=3, nype=3, ixseps1=0, ixseps2=0)
-
-        fieldperp_global_yind = 3
-        fieldperp_yproc_ind = 0
-
-        rng = np.random.default_rng(103)
-
-        # SOL
-        dump_params = [
-            (0, ["xinner", "ylower"], fieldperp_global_yind),
-            (1, ["ylower"], fieldperp_global_yind),
-            (2, ["xouter", "ylower"], fieldperp_global_yind),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner", "yupper"], -1),
-            (7, ["yupper"], -1),
-            (8, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
+        data_path, expected, fieldperp_global_yind = sol_full
+        symlink_dump_files(data_path, tmp_path)
 
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -584,43 +914,16 @@ def test_sol(self, tmp_path, squash_params, collect_kwargs):
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull_min_files(self, tmp_path, squash_params, collect_kwargs):
+    def test_singlenull_min_files(
+        self, single_null_min, tmp_path, squash_params, collect_kwargs
+    ):
         """
         Check output from a single-null case using the minimum number of processes
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(nype=3, ixseps1=4, xpoints=1)
-
-        fieldperp_global_yind = 7
-        fieldperp_yproc_ind = 1
-
-        rng = np.random.default_rng(104)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "xouter", "ylower"], -1),
-            # core
-            (1, ["xinner", "xouter"], fieldperp_global_yind),
-            # outer divertor leg
-            (2, ["xinner", "xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
+        data_path, expected, fieldperp_global_yind = single_null_min
+        symlink_dump_files(data_path, tmp_path)
 
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -635,98 +938,40 @@ def test_singlenull_min_files(self, tmp_path, squash_params, collect_kwargs):
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
     def test_singlenull_min_files_lower_boundary_fieldperp(
-        self, tmp_path, squash_params, collect_kwargs
+        self, single_null_lower_boundary, tmp_path, squash_params, collect_kwargs
     ):
         """
         Check output from a single-null case using the minimum number of processes. This
         test puts the FieldPerp in the lower boundary.
         """
+        data_path, expected, fieldperp_global_yind = single_null_lower_boundary
+        symlink_dump_files(data_path, tmp_path)
+
         squash, squash_kwargs = squash_params
 
-        grid_info = make_grid_info(nype=3, ixseps1=4, xpoints=1)
+        check_collected_data(
+            expected,
+            fieldperp_global_yind=fieldperp_global_yind,
+            doublenull=False,
+            path=tmp_path,
+            squash=squash,
+            collect_kwargs=collect_kwargs,
+            squash_kwargs=squash_kwargs,
+        )
 
-        fieldperp_global_yind = 1
-        fieldperp_yproc_ind = 0
+    @pytest.mark.parametrize("squash_params", squash_params_list)
+    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
+    def test_singlenull_min_files_upper_boundary_fieldperp(
+        self, single_null_upper_boundary, tmp_path, squash_params, collect_kwargs
+    ):
+        """
+        Check output from a single-null case using the minimum number of processes. This
+        test puts the FieldPerp in the upper boundary.
+        """
+        data_path, expected, fieldperp_global_yind = single_null_upper_boundary
+        symlink_dump_files(data_path, tmp_path)
 
-        rng = np.random.default_rng(104)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "xouter", "ylower"], fieldperp_global_yind),
-            # core
-            (1, ["xinner", "xouter"], -1),
-            # outer divertor leg
-            (2, ["xinner", "xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull_min_files_upper_boundary_fieldperp(
-        self, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a single-null case using the minimum number of processes. This
-        test puts the FieldPerp in the upper boundary.
-        """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(nype=3, ixseps1=4, xpoints=1)
-
-        fieldperp_global_yind = 14
-        fieldperp_yproc_ind = 2
-
-        rng = np.random.default_rng(104)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "xouter", "ylower"], -1),
-            # core
-            (1, ["xinner", "xouter"], -1),
-            # outer divertor leg
-            (2, ["xinner", "xouter", "yupper"], fieldperp_global_yind),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -740,49 +985,19 @@ def test_singlenull_min_files_upper_boundary_fieldperp(
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     def test_singlenull_min_files_fieldperp_on_two_yproc_different_index(
-        self, tmp_path, squash_params
+        self, single_null_inconsistent, tmp_path, squash_params
     ):
         """
         Check output from a single-null case using the minimum number of processes. This
         test has FieldPerps created with inconsistent y-indices to check this produces
         an error.
         """
-        squash, squash_kwargs = squash_params
+        data_path, expected, fieldperp_global_yind = single_null_inconsistent
+        symlink_dump_files(data_path, tmp_path)
 
+        squash, squash_kwargs = squash_params
         collect_kwargs = {"xguards": True, "yguards": "include_upper"}
 
-        grid_info = make_grid_info(nype=3, ixseps1=4, xpoints=1)
-
-        fieldperp_global_yind = 7
-        fieldperp_yproc_ind = 1
-
-        rng = np.random.default_rng(104)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "xouter", "ylower"], 2),
-            # core
-            (1, ["xinner", "xouter"], 7),
-            # outer divertor leg
-            (2, ["xinner", "xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
-
         with pytest.raises(ValueError, match="Found FieldPerp"):
             check_collected_data(
                 expected,
@@ -796,69 +1011,18 @@ def test_singlenull_min_files_fieldperp_on_two_yproc_different_index(
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull(self, tmp_path, squash_params, collect_kwargs):
+    def test_singlenull(
+        self, single_null_full, tmp_path, squash_params, collect_kwargs
+    ):
         """
         Check output from a single-null case using a large number of processes. 'Large'
         means there is at least one process in each region with no edges touching
         another region.
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(nxpe=3, nype=9, ixseps1=7, xpoints=1)
-
-        fieldperp_global_yind = 19
-        fieldperp_yproc_ind = 4
+        data_path, expected, fieldperp_global_yind = single_null_full
+        symlink_dump_files(data_path, tmp_path)
 
-        rng = np.random.default_rng(105)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "ylower"], -1),
-            (1, ["ylower"], -1),
-            (2, ["xouter", "ylower"], -1),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner"], -1),
-            (7, [], -1),
-            (8, ["xouter"], -1),
-            # core
-            (9, ["xinner"], -1),
-            (10, [], -1),
-            (11, ["xouter"], -1),
-            (12, ["xinner"], fieldperp_global_yind),
-            (13, [], fieldperp_global_yind),
-            (14, ["xouter"], fieldperp_global_yind),
-            (15, ["xinner"], -1),
-            (16, [], -1),
-            (17, ["xouter"], -1),
-            # outer divertor leg
-            (18, ["xinner"], -1),
-            (19, [], -1),
-            (20, ["xouter"], -1),
-            (21, ["xinner"], -1),
-            (22, [], -1),
-            (23, ["xouter"], -1),
-            (24, ["xinner", "yupper"], -1),
-            (25, ["yupper"], -1),
-            (26, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -872,75 +1036,18 @@ def test_singlenull(self, tmp_path, squash_params, collect_kwargs):
 
     @pytest.mark.parametrize(
         "squash_kwargs",
-        (
-            # {"parallel": False},
-            {"parallel": 1},
-            {"parallel": 2},
-            {"parallel": 8},
-            {"parallel": True},
-        ),
+        ({"parallel": 1}, {"parallel": 8}, {"parallel": True}),
     )
-    def test_singlenull_squashoutput_np(self, tmp_path, squash_kwargs):
+    def test_singlenull_squashoutput_np(
+        self, single_null_full, tmp_path, squash_kwargs
+    ):
         """
         Check output from a single-null case using a large number of processes. 'Large'
         means there is at least one process in each region with no edges touching
         another region.
         """
-        grid_info = make_grid_info(nxpe=3, nype=9, ixseps1=7, xpoints=1)
-
-        fieldperp_global_yind = 19
-        fieldperp_yproc_ind = 4
-
-        rng = np.random.default_rng(105)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "ylower"], -1),
-            (1, ["ylower"], -1),
-            (2, ["xouter", "ylower"], -1),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner"], -1),
-            (7, [], -1),
-            (8, ["xouter"], -1),
-            # core
-            (9, ["xinner"], -1),
-            (10, [], -1),
-            (11, ["xouter"], -1),
-            (12, ["xinner"], fieldperp_global_yind),
-            (13, [], fieldperp_global_yind),
-            (14, ["xouter"], fieldperp_global_yind),
-            (15, ["xinner"], -1),
-            (16, [], -1),
-            (17, ["xouter"], -1),
-            # outer divertor leg
-            (18, ["xinner"], -1),
-            (19, [], -1),
-            (20, ["xouter"], -1),
-            (21, ["xinner"], -1),
-            (22, [], -1),
-            (23, ["xouter"], -1),
-            (24, ["xinner", "yupper"], -1),
-            (25, ["yupper"], -1),
-            (26, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        data_path, expected, fieldperp_global_yind = single_null_full
+        symlink_dump_files(data_path, tmp_path)
 
         check_collected_data(
             expected,
@@ -1297,7 +1404,7 @@ def test_singlenull_squashoutput_np(self, tmp_path, squash_kwargs):
         ],
     )
     def test_singlenull_tind_xind_yind_zind(
-        self, tmp_path, squash_params, tind, xind, yind, zind
+        self, single_null_full, tmp_path, squash_params, tind, xind, yind, zind
     ):
         """
         Check output from a single-null case using a large number of processes. 'Large'
@@ -1305,6 +1412,9 @@ def test_singlenull_tind_xind_yind_zind(
         another region. This test checks the 'tind', 'xind', 'yind' and 'zind' arguments
         to `collect()` and `squashoutput()`.
         """
+        data_path, expected_original, fieldperp_global_yind = single_null_full
+        symlink_dump_files(data_path, tmp_path)
+
         tind, tslice = tind
         xind, xslice = xind
         yind, yslice = yind
@@ -1321,62 +1431,7 @@ def test_singlenull_tind_xind_yind_zind(
             "zind": zind,
         }
 
-        grid_info = make_grid_info(nxpe=3, nype=9, ixseps1=7, xpoints=1)
-
-        fieldperp_global_yind = 19
-        fieldperp_yproc_ind = 4
-
-        rng = np.random.default_rng(106)
-
-        dump_params = [
-            # inner divertor leg
-            (0, ["xinner", "ylower"], -1),
-            (1, ["ylower"], -1),
-            (2, ["xouter", "ylower"], -1),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner"], -1),
-            (7, [], -1),
-            (8, ["xouter"], -1),
-            # core
-            (9, ["xinner"], -1),
-            (10, [], -1),
-            (11, ["xouter"], -1),
-            (12, ["xinner"], fieldperp_global_yind),
-            (13, [], fieldperp_global_yind),
-            (14, ["xouter"], fieldperp_global_yind),
-            (15, ["xinner"], -1),
-            (16, [], -1),
-            (17, ["xouter"], -1),
-            # outer divertor leg
-            (18, ["xinner"], -1),
-            (19, [], -1),
-            (20, ["xouter"], -1),
-            (21, ["xinner"], -1),
-            (22, [], -1),
-            (23, ["xouter"], -1),
-            (24, ["xinner", "yupper"], -1),
-            (25, ["yupper"], -1),
-            (26, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
-
+        expected = copy.deepcopy(expected_original)
         # Can only apply here (before effect of 'xguards' and 'yguards' is applied in
         # check_collected_data) because we keep 'xguards=True' and
         # 'yguards="include_upper"' for this test, so neither has an effect.
@@ -1395,51 +1450,16 @@ def test_singlenull_tind_xind_yind_zind(
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list_full)
     def test_connected_doublenull_min_files(
-        self, tmp_path, squash_params, collect_kwargs
+        self, connected_double_null_min, tmp_path, squash_params, collect_kwargs
     ):
         """
         Check output from a connected double-null case using the minimum number of
         processes
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(nype=6, ixseps1=4, ixseps2=4, xpoints=2)
-
-        fieldperp_global_yind = 7
-        fieldperp_yproc_ind = 1
-
-        rng = np.random.default_rng(107)
-
-        dump_params = [
-            # inner, lower divertor leg
-            (0, ["xinner", "xouter", "ylower"], -1),
-            # inner core
-            (1, ["xinner", "xouter"], fieldperp_global_yind),
-            # inner, upper divertor leg
-            (2, ["xinner", "xouter", "yupper"], -1),
-            # outer, upper divertor leg
-            (3, ["xinner", "xouter", "ylower"], -1),
-            # outer core
-            (4, ["xinner", "xouter"], -1),
-            # outer, lower divertor leg
-            (5, ["xinner", "xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
+        data_path, expected, fieldperp_global_yind = connected_double_null_min
+        symlink_dump_files(data_path, tmp_path)
 
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -1453,99 +1473,18 @@ def test_connected_doublenull_min_files(
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_connected_doublenull(self, tmp_path, squash_params, collect_kwargs):
+    def test_connected_doublenull(
+        self, connected_double_null_full, tmp_path, squash_params, collect_kwargs
+    ):
         """
         Check output from a connected double-null case using a large number of
         processes. 'Large' means there is at least one process in each region with no
         edges touching another region.
         """
-        squash, squash_kwargs = squash_params
+        data_path, expected, fieldperp_global_yind = connected_double_null_full
+        symlink_dump_files(data_path, tmp_path)
 
-        grid_info = make_grid_info(nxpe=3, nype=18, ixseps1=7, ixseps2=7, xpoints=2)
-
-        fieldperp_global_yind = 19
-        fieldperp_yproc_ind = 4
-
-        rng = np.random.default_rng(108)
-
-        dump_params = [
-            # inner, lower divertor leg
-            (0, ["xinner", "ylower"], -1),
-            (1, ["ylower"], -1),
-            (2, ["xouter", "ylower"], -1),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner"], -1),
-            (7, [], -1),
-            (8, ["xouter"], -1),
-            # inner core
-            (9, ["xinner"], -1),
-            (10, [], -1),
-            (11, ["xouter"], -1),
-            (12, ["xinner"], fieldperp_global_yind),
-            (13, [], fieldperp_global_yind),
-            (14, ["xouter"], fieldperp_global_yind),
-            (15, ["xinner"], -1),
-            (16, [], -1),
-            (17, ["xouter"], -1),
-            # inner, upper divertor leg
-            (18, ["xinner"], -1),
-            (19, [], -1),
-            (20, ["xouter"], -1),
-            (21, ["xinner"], -1),
-            (22, [], -1),
-            (23, ["xouter"], -1),
-            (24, ["xinner", "yupper"], -1),
-            (25, ["yupper"], -1),
-            (26, ["xouter", "yupper"], -1),
-            # outer, upper divertor leg
-            (27, ["xinner", "ylower"], -1),
-            (28, ["ylower"], -1),
-            (29, ["xouter", "ylower"], -1),
-            (30, ["xinner"], -1),
-            (31, [], -1),
-            (32, ["xouter"], -1),
-            (33, ["xinner"], -1),
-            (34, [], -1),
-            (35, ["xouter"], -1),
-            # outer core
-            (36, ["xinner"], -1),
-            (37, [], -1),
-            (38, ["xouter"], -1),
-            (39, ["xinner"], -1),
-            (40, [], -1),
-            (41, ["xouter"], -1),
-            (42, ["xinner"], -1),
-            (43, [], -1),
-            (44, ["xouter"], -1),
-            # outer, lower divertor leg
-            (45, ["xinner"], -1),
-            (46, [], -1),
-            (47, ["xouter"], -1),
-            (48, ["xinner"], -1),
-            (49, [], -1),
-            (50, ["xouter"], -1),
-            (51, ["xinner", "yupper"], -1),
-            (52, ["yupper"], -1),
-            (53, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -1560,51 +1499,16 @@ def test_connected_doublenull(self, tmp_path, squash_params, collect_kwargs):
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
     def test_disconnected_doublenull_min_files(
-        self, tmp_path, squash_params, collect_kwargs
+        self, disconnected_double_null_min, tmp_path, squash_params, collect_kwargs
     ):
         """
         Check output from a disconnected double-null case using the minimum number of
         processes
         """
-        squash, squash_kwargs = squash_params
+        data_path, expected, fieldperp_global_yind = disconnected_double_null_min
+        symlink_dump_files(data_path, tmp_path)
 
-        grid_info = make_grid_info(nype=6, ixseps1=3, ixseps2=5, xpoints=2)
-
-        fieldperp_global_yind = 7
-        fieldperp_yproc_ind = 1
-
-        rng = np.random.default_rng(109)
-
-        dump_params = [
-            # inner, lower divertor leg
-            (0, ["xinner", "xouter", "ylower"], -1),
-            # inner core
-            (1, ["xinner", "xouter"], fieldperp_global_yind),
-            # inner, upper divertor leg
-            (2, ["xinner", "xouter", "yupper"], -1),
-            # outer, upper divertor leg
-            (3, ["xinner", "xouter", "ylower"], -1),
-            # outer core
-            (4, ["xinner", "xouter"], -1),
-            # outer, lower divertor leg
-            (5, ["xinner", "xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        squash, squash_kwargs = squash_params
 
         check_collected_data(
             expected,
@@ -1736,98 +1640,17 @@ def test_disconnected_doublenull(
             {"compress": True, "complevel": 9},
         ],
     )
-    def test_disconnected_doublenull_with_compression(self, tmp_path, squash_kwargs):
+    def test_disconnected_doublenull_with_compression(
+        self, disconnected_double_null_full, tmp_path, squash_kwargs
+    ):
         """
         Check output from a disconnected double-null case using a large number of
         processes. 'Large' means there is at least one process in each region with no
         edges touching another region. This test checks some compression options that
         can be used with `squashoutput()`, verifying that they do not modify data.
         """
-        grid_info = make_grid_info(nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2)
-
-        fieldperp_global_yind = 19
-        fieldperp_yproc_ind = 4
-
-        rng = np.random.default_rng(111)
-
-        dump_params = [
-            # inner, lower divertor leg
-            (0, ["xinner", "ylower"], -1),
-            (1, ["ylower"], -1),
-            (2, ["xouter", "ylower"], -1),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner"], -1),
-            (7, [], -1),
-            (8, ["xouter"], -1),
-            # inner core
-            (9, ["xinner"], -1),
-            (10, [], -1),
-            (11, ["xouter"], -1),
-            (12, ["xinner"], fieldperp_global_yind),
-            (13, [], fieldperp_global_yind),
-            (14, ["xouter"], fieldperp_global_yind),
-            (15, ["xinner"], -1),
-            (16, [], -1),
-            (17, ["xouter"], -1),
-            # inner, upper divertor leg
-            (18, ["xinner"], -1),
-            (19, [], -1),
-            (20, ["xouter"], -1),
-            (21, ["xinner"], -1),
-            (22, [], -1),
-            (23, ["xouter"], -1),
-            (24, ["xinner", "yupper"], -1),
-            (25, ["yupper"], -1),
-            (26, ["xouter", "yupper"], -1),
-            # outer, upper divertor leg
-            (27, ["xinner", "ylower"], -1),
-            (28, ["ylower"], -1),
-            (29, ["xouter", "ylower"], -1),
-            (30, ["xinner"], -1),
-            (31, [], -1),
-            (32, ["xouter"], -1),
-            (33, ["xinner"], -1),
-            (34, [], -1),
-            (35, ["xouter"], -1),
-            # outer core
-            (36, ["xinner"], -1),
-            (37, [], -1),
-            (38, ["xouter"], -1),
-            (39, ["xinner"], -1),
-            (40, [], -1),
-            (41, ["xouter"], -1),
-            (42, ["xinner"], -1),
-            (43, [], -1),
-            (44, ["xouter"], -1),
-            # outer, lower divertor leg
-            (45, ["xinner"], -1),
-            (46, [], -1),
-            (47, ["xouter"], -1),
-            (48, ["xinner"], -1),
-            (49, [], -1),
-            (50, ["xouter"], -1),
-            (51, ["xinner", "yupper"], -1),
-            (52, ["yupper"], -1),
-            (53, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        data_path, expected, fieldperp_global_yind = disconnected_double_null_full
+        symlink_dump_files(data_path, tmp_path)
 
         collect_kwargs = {"xguards": True, "yguards": "include_upper"}
 

From 723e32b713e36463c0f07b9611168ec31b2e3958 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 31 Jul 2024 11:25:32 +0100
Subject: [PATCH 07/17] Cut down on collect slicing tests parameterisation

We were testing slices in t, x, y, z, and all four combined. The
combined slices should really cover all the other cases, so we can get
rid of most of them. This just removes the y and z cases, but could
probably also remove the t and x ones too
---
 src/boutdata/tests/test_collect.py | 134 -----------------------------
 1 file changed, 134 deletions(-)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index dbc590a14..e08e6a553 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -1205,140 +1205,6 @@ def test_singlenull_squashoutput_np(
                 (None, slice(None)),
                 (None, slice(None)),
             ),
-            # y-slicing
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (17, slice(17, 18)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(30), slice(30)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                ([0, 28], slice(29)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(5, None), slice(5, None)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                ([6, -1], slice(6, None)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(7, 28), slice(7, 28)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                ([8, 27], slice(8, 28)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(None, None, 5), slice(None, None, 5)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                ([0, -1, 6], slice(None, -1, 6)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(9, 26, 7), slice(9, 26, 7)),
-                (None, slice(None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                ([5, 33, 4], slice(5, 33, 4)),
-                (None, slice(None)),
-            ),
-            # z-slicing
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                (1, slice(1, 2)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(3), slice(3)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                ([0, 2], slice(3)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(1, None), slice(1, None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                ([1, -1], slice(1, None)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(1, 3), slice(1, 3)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                ([1, 2], slice(1, 3)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(None, None, 2), slice(None, None, 2)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                ([0, -1, 2], slice(None, -1, 2)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                (slice(1, 4, 2), slice(1, 4, 2)),
-            ),
-            (
-                (None, slice(None)),
-                (None, slice(None)),
-                (None, slice(None)),
-                ([1, 3, 2], slice(1, 3, 2)),
-            ),
             # combined slicing
             ((2, slice(2, 3)), (7, slice(7, 8)), (17, slice(17, 18)), (1, slice(1, 2))),
             (

From b280a2770e639cabc4483716416a1f9ecbde5cc7 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 31 Jul 2024 12:08:44 +0100
Subject: [PATCH 08/17] Cache dump file creation for collect test with guard
 cell sizes

This also changes it to just parameterise `mxg` and `myg` together
---
 src/boutdata/tests/test_collect.py | 105 +++--------------------------
 1 file changed, 11 insertions(+), 94 deletions(-)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index e08e6a553..d59bbd326 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -557,7 +557,9 @@ def disconnected_double_null_min(tmp_path_factory):
 
 
 @pytest.fixture(scope="module")
-def disconnected_double_null_full(tmp_path_factory):
+def disconnected_double_null_full(tmp_path_factory, request=None):
+    mxg = request.param if request is not None else 2
+
     tmp_path = tmp_path_factory.getbasetemp() / "disconnected_double_null_full"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -626,7 +628,9 @@ def disconnected_double_null_full(tmp_path_factory):
         (53, ["xouter", "yupper"], -1),
     ]
     expected = create_dump_file_set(
-        make_grid_info(nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2),
+        make_grid_info(
+            mxg=mxg, myg=mxg, nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2
+        ),
         fieldperp_global_yind,
         tmp_path,
         np.random.default_rng(110),
@@ -1388,106 +1392,19 @@ def test_disconnected_doublenull_min_files(
 
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    @pytest.mark.parametrize("mxg", [0, 1, 2])
-    @pytest.mark.parametrize("myg", [0, 1, 2])
+    @pytest.mark.parametrize("disconnected_double_null_full", [0, 1, 2], indirect=True)
     def test_disconnected_doublenull(
-        self, tmp_path, squash_params, collect_kwargs, mxg, myg
+        self, disconnected_double_null_full, tmp_path, squash_params, collect_kwargs
     ):
         """
         Check output from a disconnected double-null case using a large number of
         processes. 'Large' means there is at least one process in each region with no
         edges touching another region.
         """
-        squash, squash_kwargs = squash_params
-
-        grid_info = make_grid_info(
-            mxg=mxg, myg=myg, nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2
-        )
-
-        fieldperp_global_yind = 19
-        fieldperp_yproc_ind = 4
-
-        rng = np.random.default_rng(110)
-
-        dump_params = [
-            # inner, lower divertor leg
-            (0, ["xinner", "ylower"], -1),
-            (1, ["ylower"], -1),
-            (2, ["xouter", "ylower"], -1),
-            (3, ["xinner"], -1),
-            (4, [], -1),
-            (5, ["xouter"], -1),
-            (6, ["xinner"], -1),
-            (7, [], -1),
-            (8, ["xouter"], -1),
-            # inner core
-            (9, ["xinner"], -1),
-            (10, [], -1),
-            (11, ["xouter"], -1),
-            (12, ["xinner"], fieldperp_global_yind),
-            (13, [], fieldperp_global_yind),
-            (14, ["xouter"], fieldperp_global_yind),
-            (15, ["xinner"], -1),
-            (16, [], -1),
-            (17, ["xouter"], -1),
-            # inner, upper divertor leg
-            (18, ["xinner"], -1),
-            (19, [], -1),
-            (20, ["xouter"], -1),
-            (21, ["xinner"], -1),
-            (22, [], -1),
-            (23, ["xouter"], -1),
-            (24, ["xinner", "yupper"], -1),
-            (25, ["yupper"], -1),
-            (26, ["xouter", "yupper"], -1),
-            # outer, upper divertor leg
-            (27, ["xinner", "ylower"], -1),
-            (28, ["ylower"], -1),
-            (29, ["xouter", "ylower"], -1),
-            (30, ["xinner"], -1),
-            (31, [], -1),
-            (32, ["xouter"], -1),
-            (33, ["xinner"], -1),
-            (34, [], -1),
-            (35, ["xouter"], -1),
-            # outer core
-            (36, ["xinner"], -1),
-            (37, [], -1),
-            (38, ["xouter"], -1),
-            (39, ["xinner"], -1),
-            (40, [], -1),
-            (41, ["xouter"], -1),
-            (42, ["xinner"], -1),
-            (43, [], -1),
-            (44, ["xouter"], -1),
-            # outer, lower divertor leg
-            (45, ["xinner"], -1),
-            (46, [], -1),
-            (47, ["xouter"], -1),
-            (48, ["xinner"], -1),
-            (49, [], -1),
-            (50, ["xouter"], -1),
-            (51, ["xinner", "yupper"], -1),
-            (52, ["yupper"], -1),
-            (53, ["xouter", "yupper"], -1),
-        ]
-        dumps = []
-        for i, boundaries, fieldperp_yind in dump_params:
-            dumps.append(
-                create_dump_file(
-                    tmpdir=tmp_path,
-                    rng=rng,
-                    grid_info=grid_info,
-                    i=i,
-                    boundaries=boundaries,
-                    fieldperp_global_yind=fieldperp_yind,
-                )
-            )
-
-        expected = concatenate_data(
-            dumps, nxpe=grid_info["NXPE"], fieldperp_yproc_ind=fieldperp_yproc_ind
-        )
+        data_path, expected, fieldperp_global_yind = disconnected_double_null_full
+        symlink_dump_files(data_path, tmp_path)
 
+        squash, squash_kwargs = squash_params
         check_collected_data(
             expected,
             fieldperp_global_yind=fieldperp_global_yind,

From 4c1f3847b86f2aea1426ffb784a2ad8a75762489 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 31 Jul 2024 12:10:26 +0100
Subject: [PATCH 09/17] Only make one of each kind of field in test dump files

The majority of `collect` tests time is spent in `datafile.open`, and
because we re-open the file for each field we collect, reducing the
number of fields in the files gives us a big speed up
---
 src/boutdata/tests/make_test_data.py | 62 +++++-----------------------
 1 file changed, 11 insertions(+), 51 deletions(-)

diff --git a/src/boutdata/tests/make_test_data.py b/src/boutdata/tests/make_test_data.py
index f8fe0a5ee..29d7ad4c5 100644
--- a/src/boutdata/tests/make_test_data.py
+++ b/src/boutdata/tests/make_test_data.py
@@ -3,13 +3,13 @@
 import numpy as np
 from netCDF4 import Dataset
 
-field3d_t_list = ["field3d_t_1", "field3d_t_2"]
-field3d_list = ["field3d_1", "field3d_2"]
-field2d_t_list = ["field2d_t_1", "field2d_t_2"]
-field2d_list = ["field2d_1", "field2d_2"]
-fieldperp_t_list = ["fieldperp_t_1", "fieldperp_t_2"]
-fieldperp_list = ["fieldperp_1", "fieldperp_2"]
-scalar_t_list = ["t_array", "scalar_t_1", "scalar_t_2"]
+field3d_t_list = ["field3d_t_1"]
+field3d_list = ["field3d_1"]
+field2d_t_list = ["field2d_t_1"]
+field2d_list = ["field2d_1"]
+fieldperp_t_list = ["fieldperp_t_1"]
+fieldperp_list = ["fieldperp_1"]
+scalar_t_list = ["t_array", "scalar_t_1"]
 
 # Note "yindex_global" attribute not included here for FieldPerps, because it is handled
 # specially
@@ -19,61 +19,31 @@
         "direction_y": "Standard",
         "direction_z": "Standard",
     },
-    "field3d_t_2": {
-        "cell_location": "CELL_CENTRE",
-        "direction_y": "Standard",
-        "direction_z": "Standard",
-    },
     "field3d_1": {
         "cell_location": "CELL_CENTRE",
         "direction_y": "Standard",
         "direction_z": "Standard",
     },
-    "field3d_2": {
-        "cell_location": "CELL_CENTRE",
-        "direction_y": "Standard",
-        "direction_z": "Standard",
-    },
     "field2d_t_1": {
         "cell_location": "CELL_CENTRE",
         "direction_y": "Standard",
         "direction_z": "Average",
     },
-    "field2d_t_2": {
-        "cell_location": "CELL_CENTRE",
-        "direction_y": "Standard",
-        "direction_z": "Average",
-    },
     "field2d_1": {
         "cell_location": "CELL_CENTRE",
         "direction_y": "Standard",
         "direction_z": "Average",
     },
-    "field2d_2": {
-        "cell_location": "CELL_CENTRE",
-        "direction_y": "Standard",
-        "direction_z": "Average",
-    },
     "fieldperp_t_1": {
         "cell_location": "CELL_CENTRE",
         "direction_y": "Standard",
         "direction_z": "Standard",
     },
-    "fieldperp_t_2": {
-        "cell_location": "CELL_CENTRE",
-        "direction_y": "Standard",
-        "direction_z": "Standard",
-    },
     "fieldperp_1": {
         "cell_location": "CELL_CENTRE",
         "direction_y": "Standard",
         "direction_z": "Standard",
     },
-    "fieldperp_2": {
-        "cell_location": "CELL_CENTRE",
-        "direction_y": "Standard",
-        "direction_z": "Standard",
-    },
 }
 
 expected_file_attributes = {
@@ -228,7 +198,6 @@ def create3D_t(name):
             result[name] = data[:, xslice, yslice, zslice]
 
         create3D_t("field3d_t_1")
-        create3D_t("field3d_t_2")
 
         def create3D(name):
             var = outputfile.createVariable(name, float, ("x", "y", "z"))
@@ -241,7 +210,6 @@ def create3D(name):
             result[name] = data[xslice, yslice, zslice]
 
         create3D("field3d_1")
-        create3D("field3d_2")
 
         # Field2D
         def create2D_t(name):
@@ -255,7 +223,6 @@ def create2D_t(name):
             result[name] = data[:, xslice, yslice]
 
         create2D_t("field2d_t_1")
-        create2D_t("field2d_t_2")
 
         def create2D(name):
             var = outputfile.createVariable(name, float, ("x", "y"))
@@ -268,7 +235,6 @@ def create2D(name):
             result[name] = data[xslice, yslice]
 
         create2D("field2d_1")
-        create2D("field2d_2")
 
         # FieldPerp
         def createPerp_t(name):
@@ -283,7 +249,6 @@ def createPerp_t(name):
             result[name] = data[:, xslice, zslice]
 
         createPerp_t("fieldperp_t_1")
-        createPerp_t("fieldperp_t_2")
 
         def createPerp(name):
             var = outputfile.createVariable(name, float, ("x", "z"))
@@ -297,7 +262,6 @@ def createPerp(name):
             result[name] = data[xslice, zslice]
 
         createPerp("fieldperp_1")
-        createPerp("fieldperp_2")
 
         # Time-dependent array
         def createScalar_t(name):
@@ -310,7 +274,6 @@ def createScalar_t(name):
 
         createScalar_t("t_array")
         createScalar_t("scalar_t_1")
-        createScalar_t("scalar_t_2")
 
         # Scalar
         def createScalar(name, value):
@@ -388,7 +351,6 @@ def create3D(name):
             result[name] = data[xslice, yslice, zslice]
 
         create3D("field3d_1")
-        create3D("field3d_2")
 
         # Field2D
         def create2D(name):
@@ -402,7 +364,6 @@ def create2D(name):
             result[name] = data[xslice, yslice]
 
         create2D("field2d_1")
-        create2D("field2d_2")
 
         # FieldPerp
         def createPerp(name):
@@ -417,7 +378,6 @@ def createPerp(name):
             result[name] = data[xslice, zslice]
 
         createPerp("fieldperp_1")
-        createPerp("fieldperp_2")
 
         # Scalar
         def createScalar(name, value):
@@ -466,7 +426,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
         raise ValueError("nxpe={} does not divide len(data_list)={}".format(nxpe, npes))
 
     if has_t_dim:
-        for var in ("field3d_t_1", "field3d_t_2", "field2d_t_1", "field2d_t_2"):
+        for var in ("field3d_t_1", "field2d_t_1"):
             # Join in x-direction
             parts = [
                 np.concatenate(
@@ -477,7 +437,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
             # Join in y-direction
             result[var] = np.concatenate(parts, axis=2)
 
-    for var in ("field3d_1", "field3d_2", "field2d_1", "field2d_2"):
+    for var in ("field3d_1", "field2d_1"):
         # Join in x-direction
         parts = [
             np.concatenate(
@@ -489,7 +449,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
         result[var] = np.concatenate(parts, axis=1)
 
     if has_t_dim:
-        for var in ("fieldperp_t_1", "fieldperp_t_2"):
+        for var in ("fieldperp_t_1",):
             # Join in x-direction
             result[var] = np.concatenate(
                 [
@@ -501,7 +461,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
                 axis=1,
             )
 
-    for var in ("fieldperp_1", "fieldperp_2"):
+    for var in ("fieldperp_1",):
         # Join in x-direction
         result[var] = np.concatenate(
             [

From 55002e2da72a95e0e8c3327fd1b1665c1d2a9568 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 31 Jul 2024 17:04:13 +0100
Subject: [PATCH 10/17] Replace duplicated collect tests with parameterised
 test

---
 src/boutdata/tests/test_collect.py | 505 ++++++++++-------------------
 1 file changed, 169 insertions(+), 336 deletions(-)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index d59bbd326..6140cea31 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -156,6 +156,8 @@ def create_dump_file_set(
 
 @pytest.fixture(scope="module")
 def core_min(tmp_path_factory):
+    """Check output from a core-only case using the minimum number of processes"""
+
     tmp_path = tmp_path_factory.getbasetemp() / "core_min"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -180,6 +182,11 @@ def core_min(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def core_full(tmp_path_factory):
+    """
+    Check output from a core-only case using a large number of processes. 'Large'
+    means there is at least one process in each region with no edges touching
+    another region.
+    """
     tmp_path = tmp_path_factory.getbasetemp() / "core_full"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -211,6 +218,8 @@ def core_full(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def sol_min(tmp_path_factory):
+    """Check output from a SOL-only case using the minimum number of processes"""
+
     tmp_path = tmp_path_factory.getbasetemp() / "sol_min"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -232,6 +241,11 @@ def sol_min(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def sol_full(tmp_path_factory):
+    """
+    Check output from a SOL-only case using a large number of processes. 'Large'
+    means there is at least one process in each region with no edges touching
+    another region.
+    """
     tmp_path = tmp_path_factory.getbasetemp() / "sol_full"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -261,6 +275,8 @@ def sol_full(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def single_null_min(tmp_path_factory):
+    """Check output from a single-null case using the minimum number of processes"""
+
     tmp_path = tmp_path_factory.getbasetemp() / "single_null_min"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -287,6 +303,11 @@ def single_null_min(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def single_null_lower_boundary(tmp_path_factory):
+    """
+    Check output from a single-null case using the minimum number of processes. This
+    test puts the FieldPerp in the lower boundary.
+    """
+
     tmp_path = tmp_path_factory.getbasetemp() / "single_null_lower_boundary"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -312,6 +333,11 @@ def single_null_lower_boundary(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def single_null_upper_boundary(tmp_path_factory):
+    """
+    Check output from a single-null case using the minimum number of processes. This
+    test puts the FieldPerp in the upper boundary.
+    """
+
     tmp_path = tmp_path_factory.getbasetemp() / "single_null_upper_boundary"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -338,6 +364,12 @@ def single_null_upper_boundary(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def single_null_inconsistent(tmp_path_factory):
+    """
+    Check output from a single-null case using the minimum number of processes. This
+    test has FieldPerps created with inconsistent y-indices to check this produces
+    an error.
+    """
+
     tmp_path = tmp_path_factory.getbasetemp() / "single_null_inconsistent"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -364,6 +396,11 @@ def single_null_inconsistent(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def single_null_full(tmp_path_factory):
+    """
+    Check output from a single-null case using a large number of processes. 'Large'
+    means there is at least one process in each region with no edges touching
+    another region.
+    """
     tmp_path = tmp_path_factory.getbasetemp() / "single_null_full"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -414,6 +451,8 @@ def single_null_full(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def connected_double_null_min(tmp_path_factory):
+    """Check output from a connected double-null case using the minimum number of processes"""
+
     tmp_path = tmp_path_factory.getbasetemp() / "connected_double_null_min"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -446,6 +485,12 @@ def connected_double_null_min(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def connected_double_null_full(tmp_path_factory):
+    """
+    Check output from a connected double-null case using a large number of
+    processes. 'Large' means there is at least one process in each region with no
+    edges touching another region.
+    """
+
     tmp_path = tmp_path_factory.getbasetemp() / "connected_double_null_full"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -526,6 +571,8 @@ def connected_double_null_full(tmp_path_factory):
 
 @pytest.fixture(scope="module")
 def disconnected_double_null_min(tmp_path_factory):
+    """Check output from a disconnected double-null case using the minimum number of processes"""
+
     tmp_path = tmp_path_factory.getbasetemp() / "disconnected_double_null_min"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
@@ -556,85 +603,115 @@ def disconnected_double_null_min(tmp_path_factory):
     return tmp_path, expected, fieldperp_global_yind
 
 
+disconnected_double_null_params = [
+    # inner, lower divertor leg
+    (0, ["xinner", "ylower"], -1),
+    (1, ["ylower"], -1),
+    (2, ["xouter", "ylower"], -1),
+    (3, ["xinner"], -1),
+    (4, [], -1),
+    (5, ["xouter"], -1),
+    (6, ["xinner"], -1),
+    (7, [], -1),
+    (8, ["xouter"], -1),
+    # inner core
+    (9, ["xinner"], -1),
+    (10, [], -1),
+    (11, ["xouter"], -1),
+    (12, ["xinner"], 19),
+    (13, [], 19),
+    (14, ["xouter"], 19),
+    (15, ["xinner"], -1),
+    (16, [], -1),
+    (17, ["xouter"], -1),
+    # inner, upper divertor leg
+    (18, ["xinner"], -1),
+    (19, [], -1),
+    (20, ["xouter"], -1),
+    (21, ["xinner"], -1),
+    (22, [], -1),
+    (23, ["xouter"], -1),
+    (24, ["xinner", "yupper"], -1),
+    (25, ["yupper"], -1),
+    (26, ["xouter", "yupper"], -1),
+    # outer, upper divertor leg
+    (27, ["xinner", "ylower"], -1),
+    (28, ["ylower"], -1),
+    (29, ["xouter", "ylower"], -1),
+    (30, ["xinner"], -1),
+    (31, [], -1),
+    (32, ["xouter"], -1),
+    (33, ["xinner"], -1),
+    (34, [], -1),
+    (35, ["xouter"], -1),
+    # outer core
+    (36, ["xinner"], -1),
+    (37, [], -1),
+    (38, ["xouter"], -1),
+    (39, ["xinner"], -1),
+    (40, [], -1),
+    (41, ["xouter"], -1),
+    (42, ["xinner"], -1),
+    (43, [], -1),
+    (44, ["xouter"], -1),
+    # outer, lower divertor leg
+    (45, ["xinner"], -1),
+    (46, [], -1),
+    (47, ["xouter"], -1),
+    (48, ["xinner"], -1),
+    (49, [], -1),
+    (50, ["xouter"], -1),
+    (51, ["xinner", "yupper"], -1),
+    (52, ["yupper"], -1),
+    (53, ["xouter", "yupper"], -1),
+]
+
+
 @pytest.fixture(scope="module")
-def disconnected_double_null_full(tmp_path_factory, request=None):
-    mxg = request.param if request is not None else 2
+def disconnected_double_null_full(tmp_path_factory):
+    """Check output from a disconnected double-null case using a large number of
+    processes. 'Large' means there is at least one process in each region with
+    no edges touching another region.
+
+    """
 
     tmp_path = tmp_path_factory.getbasetemp() / "disconnected_double_null_full"
     tmp_path.mkdir(parents=True, exist_ok=True)
 
     fieldperp_global_yind = 19
 
-    dump_params = [
-        # inner, lower divertor leg
-        (0, ["xinner", "ylower"], -1),
-        (1, ["ylower"], -1),
-        (2, ["xouter", "ylower"], -1),
-        (3, ["xinner"], -1),
-        (4, [], -1),
-        (5, ["xouter"], -1),
-        (6, ["xinner"], -1),
-        (7, [], -1),
-        (8, ["xouter"], -1),
-        # inner core
-        (9, ["xinner"], -1),
-        (10, [], -1),
-        (11, ["xouter"], -1),
-        (12, ["xinner"], fieldperp_global_yind),
-        (13, [], fieldperp_global_yind),
-        (14, ["xouter"], fieldperp_global_yind),
-        (15, ["xinner"], -1),
-        (16, [], -1),
-        (17, ["xouter"], -1),
-        # inner, upper divertor leg
-        (18, ["xinner"], -1),
-        (19, [], -1),
-        (20, ["xouter"], -1),
-        (21, ["xinner"], -1),
-        (22, [], -1),
-        (23, ["xouter"], -1),
-        (24, ["xinner", "yupper"], -1),
-        (25, ["yupper"], -1),
-        (26, ["xouter", "yupper"], -1),
-        # outer, upper divertor leg
-        (27, ["xinner", "ylower"], -1),
-        (28, ["ylower"], -1),
-        (29, ["xouter", "ylower"], -1),
-        (30, ["xinner"], -1),
-        (31, [], -1),
-        (32, ["xouter"], -1),
-        (33, ["xinner"], -1),
-        (34, [], -1),
-        (35, ["xouter"], -1),
-        # outer core
-        (36, ["xinner"], -1),
-        (37, [], -1),
-        (38, ["xouter"], -1),
-        (39, ["xinner"], -1),
-        (40, [], -1),
-        (41, ["xouter"], -1),
-        (42, ["xinner"], -1),
-        (43, [], -1),
-        (44, ["xouter"], -1),
-        # outer, lower divertor leg
-        (45, ["xinner"], -1),
-        (46, [], -1),
-        (47, ["xouter"], -1),
-        (48, ["xinner"], -1),
-        (49, [], -1),
-        (50, ["xouter"], -1),
-        (51, ["xinner", "yupper"], -1),
-        (52, ["yupper"], -1),
-        (53, ["xouter", "yupper"], -1),
-    ]
     expected = create_dump_file_set(
-        make_grid_info(
-            mxg=mxg, myg=mxg, nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2
-        ),
+        make_grid_info(nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2),
         fieldperp_global_yind,
         tmp_path,
         np.random.default_rng(110),
-        dump_params,
+        disconnected_double_null_params,
+        fieldperp_yproc_ind=4,
+    )
+    return tmp_path, expected, fieldperp_global_yind
+
+
+@pytest.fixture(scope="module")
+def disconnected_double_null_full_no_guards(tmp_path_factory):
+    """Check output from a disconnected double-null case using a large number of
+    processes. 'Large' means there is at least one process in each region with
+    no edges touching another region. This case has no guard cells
+
+    """
+
+    tmp_path = (
+        tmp_path_factory.getbasetemp() / "disconnected_double_null_full_no_guards"
+    )
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    fieldperp_global_yind = 19
+
+    expected = create_dump_file_set(
+        make_grid_info(mxg=0, myg=0, nxpe=3, nype=18, ixseps1=6, ixseps2=11, xpoints=2),
+        fieldperp_global_yind,
+        tmp_path,
+        np.random.default_rng(110),
+        disconnected_double_null_params,
         fieldperp_yproc_ind=4,
     )
     return tmp_path, expected, fieldperp_global_yind
@@ -690,21 +767,39 @@ def check_variable(
         assert actual.attributes["bout_type"] == "scalar"
 
 
+scenario_list = [
+    "core_min",
+    "core_full",
+    "sol_min",
+    "sol_full",
+    "single_null_min",
+    "single_null_lower_boundary",
+    "single_null_upper_boundary",
+    "single_null_full",
+    "connected_double_null_min",
+    "connected_double_null_full",
+    "disconnected_double_null_min",
+    "disconnected_double_null_full",
+    "disconnected_double_null_full_no_guards",
+]
+
+
 class TestCollect:
     @pytest.mark.parametrize("squash_params", squash_params_list)
     @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_core_min_files(self, core_min, tmp_path, squash_params, collect_kwargs):
-        """
-        Check output from a core-only case using the minimum number of processes
-        """
+    @pytest.mark.parametrize("scenario", scenario_list)
+    def test_tokamak_scenario_collect(
+        self, tmp_path, scenario, collect_kwargs, squash_params, request
+    ):
+        """Test basic collect in different scenarios"""
         squash, squash_kwargs = squash_params
-        data_path, expected, fieldperp_global_yind = core_min
+        data_path, expected, fieldperp_global_yind = request.getfixturevalue(scenario)
         symlink_dump_files(data_path, tmp_path)
 
         check_collected_data(
             expected,
             fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
+            doublenull="double_null" in scenario,
             path=tmp_path,
             squash=squash,
             collect_kwargs=collect_kwargs,
@@ -849,144 +944,6 @@ def test_core_min_files_append_time_split_raises(self, core_min, tmp_path):
                 tmp_path, outputname="boutdata.nc", **collect_kwargs, **squash_kwargs
             )
 
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_core(self, core_full, tmp_path, squash_params, collect_kwargs):
-        """
-        Check output from a core-only case using a large number of processes. 'Large'
-        means there is at least one process in each region with no edges touching
-        another region.
-        """
-        data_path, expected, fieldperp_global_yind = core_full
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_sol_min_files(self, sol_min, tmp_path, squash_params, collect_kwargs):
-        """
-        Check output from a SOL-only case using the minimum number of processes
-        """
-        data_path, expected, fieldperp_global_yind = sol_min
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_sol(self, sol_full, tmp_path, squash_params, collect_kwargs):
-        """
-        Check output from a SOL-only case using a large number of processes. 'Large'
-        means there is at least one process in each region with no edges touching
-        another region.
-        """
-        data_path, expected, fieldperp_global_yind = sol_full
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull_min_files(
-        self, single_null_min, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a single-null case using the minimum number of processes
-        """
-        data_path, expected, fieldperp_global_yind = single_null_min
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull_min_files_lower_boundary_fieldperp(
-        self, single_null_lower_boundary, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a single-null case using the minimum number of processes. This
-        test puts the FieldPerp in the lower boundary.
-        """
-        data_path, expected, fieldperp_global_yind = single_null_lower_boundary
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull_min_files_upper_boundary_fieldperp(
-        self, single_null_upper_boundary, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a single-null case using the minimum number of processes. This
-        test puts the FieldPerp in the upper boundary.
-        """
-        data_path, expected, fieldperp_global_yind = single_null_upper_boundary
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
     @pytest.mark.parametrize("squash_params", squash_params_list)
     def test_singlenull_min_files_fieldperp_on_two_yproc_different_index(
         self, single_null_inconsistent, tmp_path, squash_params
@@ -1013,31 +970,6 @@ def test_singlenull_min_files_fieldperp_on_two_yproc_different_index(
                 squash_kwargs=squash_kwargs,
             )
 
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_singlenull(
-        self, single_null_full, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a single-null case using a large number of processes. 'Large'
-        means there is at least one process in each region with no edges touching
-        another region.
-        """
-        data_path, expected, fieldperp_global_yind = single_null_full
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=False,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
     @pytest.mark.parametrize(
         "squash_kwargs",
         ({"parallel": 1}, {"parallel": 8}, {"parallel": True}),
@@ -1317,108 +1249,9 @@ def test_singlenull_tind_xind_yind_zind(
             squash_kwargs=squash_kwargs,
         )
 
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list_full)
-    def test_connected_doublenull_min_files(
-        self, connected_double_null_min, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a connected double-null case using the minimum number of
-        processes
-        """
-        data_path, expected, fieldperp_global_yind = connected_double_null_min
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=True,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_connected_doublenull(
-        self, connected_double_null_full, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a connected double-null case using a large number of
-        processes. 'Large' means there is at least one process in each region with no
-        edges touching another region.
-        """
-        data_path, expected, fieldperp_global_yind = connected_double_null_full
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=True,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    def test_disconnected_doublenull_min_files(
-        self, disconnected_double_null_min, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a disconnected double-null case using the minimum number of
-        processes
-        """
-        data_path, expected, fieldperp_global_yind = disconnected_double_null_min
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=True,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
-    @pytest.mark.parametrize("squash_params", squash_params_list)
-    @pytest.mark.parametrize("collect_kwargs", collect_kwargs_list)
-    @pytest.mark.parametrize("disconnected_double_null_full", [0, 1, 2], indirect=True)
-    def test_disconnected_doublenull(
-        self, disconnected_double_null_full, tmp_path, squash_params, collect_kwargs
-    ):
-        """
-        Check output from a disconnected double-null case using a large number of
-        processes. 'Large' means there is at least one process in each region with no
-        edges touching another region.
-        """
-        data_path, expected, fieldperp_global_yind = disconnected_double_null_full
-        symlink_dump_files(data_path, tmp_path)
-
-        squash, squash_kwargs = squash_params
-        check_collected_data(
-            expected,
-            fieldperp_global_yind=fieldperp_global_yind,
-            doublenull=True,
-            path=tmp_path,
-            squash=squash,
-            collect_kwargs=collect_kwargs,
-            squash_kwargs=squash_kwargs,
-        )
-
     @pytest.mark.parametrize(
         "squash_kwargs",
         [
-            {},
             {"compress": True, "complevel": 1},
             {"compress": True, "complevel": 9},
         ],

From f1bd1794bbc35507607a83a2b2d01fcaf55f0fa5 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 31 Jul 2024 17:04:28 +0100
Subject: [PATCH 11/17] Name squash/collect parameters for collect tests

---
 src/boutdata/tests/test_collect.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py
index 6140cea31..e6f3b2da1 100644
--- a/src/boutdata/tests/test_collect.py
+++ b/src/boutdata/tests/test_collect.py
@@ -24,8 +24,8 @@
 # Note - using tmp_path fixture requires pytest>=3.9.0
 
 collect_kwargs_list = [
-    {"xguards": True, "yguards": "include_upper"},
-    {"xguards": False, "yguards": False},
+    pytest.param({"xguards": True, "yguards": "include_upper"}, id="collect_guards"),
+    pytest.param({"xguards": False, "yguards": False}, id="collect_noguards"),
 ]
 
 collect_kwargs_list_full = [
@@ -39,9 +39,9 @@
 
 
 squash_params_list = [
-    (False, {}),
-    (True, {}),
-    (True, {"parallel": 2}),
+    pytest.param((False, {}), id="squash_off"),
+    pytest.param((True, {}), id="squash_serial"),
+    pytest.param((True, {"parallel": 2}), id="squash_parallel"),
 ]
 
 

From 90c6d668aab4a77934b24ea2af54c708a1f15228 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 17 Jul 2024 14:16:27 +0100
Subject: [PATCH 12/17] Replace geqdsk reader with FreeQDSK

---
 pyproject.toml          |   1 +
 src/boututils/geqdsk.py | 252 +++++++++++++++-------------------------
 2 files changed, 92 insertions(+), 161 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 2f8353000..939360f1d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,6 +37,7 @@ dependencies = [
     "natsort>=8.1.0",
     "scipy>=1.4.1",
     "netCDF4",
+    "freeqdsk>=0.4.0"
 ]
 dynamic = ["version"]
 
diff --git a/src/boututils/geqdsk.py b/src/boututils/geqdsk.py
index 16d8b96f1..d7b1ead77 100644
--- a/src/boututils/geqdsk.py
+++ b/src/boututils/geqdsk.py
@@ -1,20 +1,15 @@
-import re
-
-import numpy
-
 """
-@brief G-Eqdsk reader class
-@version $Id$
-
-Copyright &copy; 2006-2008, Tech-X Corporation, Boulder, CO
-See LICENSE file for conditions of use.
+G-Eqdsk reader class
 
 The official document describing g-eqdsk files:
 https://fusion.gat.com/conferences/snowmass/working/mfe/physics/p3/equilibria/g_eqdsk_s.pdf
 """
 
+import freeqdsk
+import numpy
+
 
-class Geqdsk(object):
+class Geqdsk:
     def __init__(self):
         """
         Constructor
@@ -26,155 +21,91 @@ def openFile(self, filename):
         open geqdsk file and parse its content
         """
 
-        lines = open(filename, "r").readlines()
-
-        # first line
-        m = re.search(r"^\s*(.*)\s+\d+\s+(\d+)\s+(\d+)\s*$", lines[0])
-        self.data["case"] = m.group(1), "Identification character string"
-        self.data["nw"] = int(m.group(2)), "Number of horizontal R grid points"
-        self.data["nh"] = int(m.group(3)), "Number of vertical Z grid points"
-
-        fltsPat = (
-            r"^\s*([ \-]\d\.\d+[Ee][\+\-]\d\d)([ \-]\d\.\d+[Ee][\+\-]\d\d)"
-            r"([ \-]\d\.\d+[Ee][\+\-]\d\d)([ \-]\d\.\d+[Ee][\+\-]\d\d)"
-            r"([ \-]\d\.\d+[Ee][\+\-]\d\d)\s*$"
-        )
-
-        # 2nd line
-        m = re.search(fltsPat, lines[1])
-        self.data["rdim"] = (
-            float(m.group(1)),
-            "Horizontal dimension in meter of computational box",
-        )
-        self.data["zdim"] = (
-            float(m.group(2)),
-            "Vertical dimension in meter of computational box",
-        )
-        self.data["rcentr"] = (
-            float(m.group(3)),
-            "R in meter of vacuum toroidal magnetic field BCENTR",
-        )
-        self.data["rleft"] = (
-            float(m.group(4)),
-            "Minimum R in meter of rectangular computational box",
-        )
-        self.data["zmid"] = (
-            float(m.group(5)),
-            "Z of center of computational box in meter",
-        )
-
-        # 3rd line
-        m = re.search(fltsPat, lines[2])
-        self.data["rmaxis"] = float(m.group(1)), "R of magnetic axis in meter"
-        self.data["zmaxis"] = float(m.group(2)), "Z of magnetic axis in meter"
-        self.data["simag"] = (
-            float(m.group(3)),
-            "poloidal flux at magnetic axis in Weber /rad",
-        )
-        self.data["sibry"] = (
-            float(m.group(4)),
-            "poloidal flux at the plasma boundary in Weber /rad",
-        )
-        self.data["bcentr"] = (
-            float(m.group(5)),
-            "Vacuum toroidal magnetic field in Tesla at RCENTR",
-        )
-
-        # 4th line
-        m = re.search(fltsPat, lines[3])
-        self.data["current"] = float(m.group(1)), "Plasma current in Ampere"
-        # self.data['simag'] = float(m.group(2)), ""
-
-        # self.data['rmaxis'] = float(m.group(4)), ""
-
-        # 5th line
-        m = re.search(fltsPat, lines[4])
-        # self.data['zmaxis'] = float(m.group(1)), ""
-
-        # self.data['sibry'] = float(m.group(3)), ""
-
-        # read remaining data
-        data = []
-        counter = 5
-        while 1:
-            line = lines[counter]
-            m = re.match(r"^\s*[ \-]\d\.\d+[Ee][\+\-]\d\d", line)
-            if not m:
-                break
-            data += eval("[" + re.sub(r"(\d)([ \-]\d\.)", "\\1,\\2", line) + "]")
-            counter += 1
-
-        nw = self.data["nw"][0]
-        nh = self.data["nh"][0]
-
-        self.data["fpol"] = (
-            numpy.array(data[0:nw]),
-            "Poloidal current function in m-T, F = RBT on flux grid",
-        )
-        self.data["pres"] = (
-            numpy.array(data[nw : 2 * nw]),
-            "Plasma pressure in nt / m 2 on uniform flux grid",
-        )
-
-        self.data["ffprime"] = (
-            numpy.array(data[2 * nw : 3 * nw]),
-            "FF'(psi) in (mT)^2/(Weber/rad) on uniform flux grid",
-        )
-
-        self.data["pprime"] = (
-            numpy.array(data[3 * nw : 4 * nw]),
-            "P'(psi) in (nt/m2)/(Weber/rad) on uniform flux grid",
-        )
-
-        self.data["psirz"] = (
-            numpy.reshape(data[4 * nw : 4 * nw + nw * nh], (nh, nw)),
-            "Poloidal flux in Weber / rad on the rectangular grid points",
-        )
-        self.data["qpsi"] = (
-            numpy.array(data[4 * nw + nw * nh : 5 * nw + nw * nh]),
-            "q values on uniform flux grid from axis to boundary",
-        )
-
-        line = lines[counter]
-        m = re.search(r"^\s*(\d+)\s+(\d+)", line)
-        print(line)
-        nbbbs = int(m.group(1))
-        limitr = int(m.group(2))
-        self.data["nbbbs"] = nbbbs, "Number of boundary points"
-        self.data["limitr"] = limitr, "Number of limiter points"
-        counter += 1
-
-        data = []
-        while 1:
-            line = lines[counter]
-            m = re.search(r"^\s*[ \-]\d\.\d+[Ee][\+\-]\d\d", line)
-            counter += 1
-            if not m:
-                break
-            data += eval("[" + re.sub(r"(\d)([ \-]\d\.)", "\\1,\\2", line) + "]")
-        self.data["rbbbs"] = (
-            numpy.zeros((nbbbs,), numpy.float64),
-            "R of boundary points in meter",
-        )
-        self.data["zbbbs"] = (
-            numpy.zeros((nbbbs,), numpy.float64),
-            "Z of boundary points in meter",
-        )
-        for i in range(nbbbs):
-            self.data["rbbbs"][0][i] = data[2 * i]
-            self.data["zbbbs"][0][i] = data[2 * i + 1]
-
-        self.data["rlim"] = (
-            numpy.zeros((limitr,), numpy.float64),
-            "R of surrounding limiter contour in meter",
-        )
-        self.data["zlim"] = (
-            numpy.zeros((limitr,), numpy.float64),
-            "Z of surrounding limiter contour in meter",
-        )
-        for i in range(limitr):
-            self.data["rlim"][0][i] = data[2 * nbbbs + 2 * i]
-            self.data["zlim"][0][i] = data[2 * nbbbs + 2 * i + 1]
+        with open(filename, "r") as f:
+            data = freeqdsk.geqdsk.read(f)
+
+        self.data = {
+            "case": (data["comment"], "Identification character string"),
+            "nw": (data["nw"], "Number of horizontal R grid points"),
+            "nh": (data["nh"], "Number of vertical Z grid points"),
+            "rdim": (
+                data["rdim"],
+                "Horizontal dimension in meter of computational box",
+            ),
+            "zdim": (
+                data["zdim"],
+                "Vertical dimension in meter of computational box",
+            ),
+            "rcentr": (
+                data["rcentr"],
+                "R in meter of vacuum toroidal magnetic field BCENTR",
+            ),
+            "rleft": (
+                data["rleft"],
+                "Minimum R in meter of rectangular computational box",
+            ),
+            "zmid": (
+                data["zmid"],
+                "Z of center of computational box in meter",
+            ),
+            "rmaxis": (data["rmagx"], "R of magnetic axis in meter"),
+            "zmaxis": (data["zmagx"], "Z of magnetic axis in meter"),
+            "simag": (
+                data["simagx"],
+                "poloidal flux at magnetic axis in Weber /rad",
+            ),
+            "sibry": (
+                data["sibdry"],
+                "poloidal flux at the plasma boundary in Weber /rad",
+            ),
+            "bcentr": (
+                data["bcentr"],
+                "Vacuum toroidal magnetic field in Tesla at RCENTR",
+            ),
+            "current": (data["cpasma"], "Plasma current in Ampere"),
+            "fpol": (
+                data["fpol"],
+                "Poloidal current function in m-T, F = RBT on flux grid",
+            ),
+            "pres": (
+                data["pres"],
+                "Plasma pressure in nt / m 2 on uniform flux grid",
+            ),
+            "ffprime": (
+                data["ffprime"],
+                "FF'(psi), in (mT),^2/(Weber/rad), on uniform flux grid",
+            ),
+            "pprime": (
+                data["pprime"],
+                "P'(psi), in (nt/m2),/(Weber/rad), on uniform flux grid",
+            ),
+            "psirz": (
+                data["psi"],
+                "Poloidal flux in Weber / rad on the rectangular grid points",
+            ),
+            "qpsi": (
+                data["qpsi"],
+                "q values on uniform flux grid from axis to boundary",
+            ),
+            "nbbbs": (data["nbdry"], "Number of boundary points"),
+            "limitr": (data["nlim"], "Number of limiter points"),
+            "rbbbs": (
+                data["rbdry"],
+                "R of boundary points in meter",
+            ),
+            "zbbbs": (
+                data["zbdry"],
+                "Z of boundary points in meter",
+            ),
+            "rlim": (
+                data["rlim"],
+                "R of surrounding limiter contour in meter",
+            ),
+            "zlim": (
+                data["zlim"],
+                "Z of surrounding limiter contour in meter",
+            ),
+        }
 
     def getAll(self):
         return self.data
@@ -244,7 +175,7 @@ def main():
         vs = options.vars.split(",")
 
     for v in vs:
-        print("%s: %s" % (v, str(geq.get(v))))
+        print(f"{v}: {geq.get(v)}")
 
     if options.plot:
         from matplotlib import pylab
@@ -284,7 +215,6 @@ def main():
         pylab.figure()
         pylab.pcolor(rs, zs, geq.get("psirz"), shading="interp")
         pylab.plot(geq.get("rbbbs"), geq.get("zbbbs"), "w-")
-        # pylab.plot(geq.get('rlim'), geq.get('zlim'), 'k--')
         pylab.axis("image")
         pylab.title("poloidal flux")
         pylab.xlabel("R")

From 2b4d3dcf98c17b7a68a8a52fe8ab2753d5dbeb12 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 17 Jul 2024 14:24:01 +0100
Subject: [PATCH 13/17] Open file in `Geqdsk` constructor, add slicing

---
 src/boututils/geqdsk.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/boututils/geqdsk.py b/src/boututils/geqdsk.py
index d7b1ead77..969fbd6d0 100644
--- a/src/boututils/geqdsk.py
+++ b/src/boututils/geqdsk.py
@@ -10,12 +10,14 @@
 
 
 class Geqdsk:
-    def __init__(self):
+    def __init__(self, filename: str):
         """
         Constructor
         """
         self.data = {}
 
+        self.openFile(filename)
+
     def openFile(self, filename):
         """
         open geqdsk file and parse its content
@@ -116,6 +118,9 @@ def getAllVars(self):
     def get(self, varname):
         return self.data[varname.lower()][0]
 
+    def __getitem__(self, item):
+        return self.get(item)
+
     def getDescriptor(self, varname):
         return self.data[varname.lower()][1]
 

From 4d43aba7e547eaebdaace5e56189e810f512f47a Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Wed, 17 Jul 2024 14:24:33 +0100
Subject: [PATCH 14/17] Simplify `read_geqdsk`

---
 src/boututils/read_geqdsk.py | 93 +++++++++++-------------------------
 1 file changed, 28 insertions(+), 65 deletions(-)

diff --git a/src/boututils/read_geqdsk.py b/src/boututils/read_geqdsk.py
index e804c59a9..09626672a 100644
--- a/src/boututils/read_geqdsk.py
+++ b/src/boututils/read_geqdsk.py
@@ -1,61 +1,28 @@
 import numpy
 
-from boututils.bunch import Bunch
-from boututils.geqdsk import Geqdsk
+from .bunch import Bunch
+from .geqdsk import Geqdsk
 
 
-def read_geqdsk(file):
-    data = Geqdsk()
+def read_geqdsk(filename: str):
+    data = Geqdsk(filename)
 
-    data.openFile(file)
+    nxefit = data["nw"]
+    nyefit = data["nh"]
+    xdim = data["rdim"]
+    zdim = data["zdim"]
+    rgrid1 = data["rleft"]
+    zmid = data["zmid"]
 
-    nxefit = data.get("nw")
-    nyefit = data.get("nh")
-    xdim = data.get("rdim")
-    zdim = data.get("zdim")
-    rcentr = data.get("rcentr")
-    rgrid1 = data.get("rleft")
-    zmid = data.get("zmid")
-
-    rmagx = data.get("rmaxis")
-    zmagx = data.get("zmaxis")
-    simagx = data.get("simag")
-    sibdry = data.get("sibry")
-    bcentr = data.get("bcentr")
-
-    cpasma = data.get("current")
-    # simagx =data.get('simag')
-    # xdum   =data.get()
-    # rmagx  =data.get('rmaxis')
-    # xdum   =data.get()
-
-    # zmagx  =data.get('zmaxis')
-    # xdum   =data.get()
-    # sibdry =data.get('sibry')
-    # xdum   =data.get()
-    # xdum   =data.get()
-
-    # Read arrays
-
-    fpol = data.get("fpol")
-    pres = data.get("pres")
-
-    f = data.get("psirz")
-    qpsi = data.get("qpsi")
-
-    nbdry = data.get("nbbbs")
-    nlim = data.get("limitr")
+    nlim = data["limitr"]
 
     if nlim != 0:
-        xlim = data.get("rlim")
-        ylim = data.get("zlim")
+        xlim = data["rlim"]
+        ylim = data["zlim"]
     else:
         xlim = [0]
         ylim = [0]
 
-    rbdry = data.get("rbbbs")
-    zbdry = data.get("zbbbs")
-
     # Reconstruct the (R,Z) mesh
     r = numpy.zeros((nxefit, nyefit), numpy.float64)
     z = numpy.zeros((nxefit, nyefit), numpy.float64)
@@ -65,10 +32,6 @@ def read_geqdsk(file):
             r[i, j] = rgrid1 + xdim * i / (nxefit - 1)
             z[i, j] = (zmid - 0.5 * zdim) + zdim * j / (nyefit - 1)
 
-    f = f.T
-
-    print("nxefit =  ", nxefit, "  nyefit=  ", nyefit)
-
     return Bunch(
         nx=nxefit,
         ny=nyefit,  # Number of horizontal and vertical points
@@ -76,23 +39,23 @@ def read_geqdsk(file):
         z=z,  # Location of the grid-points
         xdim=xdim,
         zdim=zdim,  # Size of the domain in meters
-        rcentr=rcentr,
-        bcentr=bcentr,  # Reference vacuum toroidal field (m, T)
+        rcentr=data["rcentr"],
+        bcentr=data["bcentr"],  # Reference vacuum toroidal field (m, T)
         rgrid1=rgrid1,  # R of left side of domain
         zmid=zmid,  # Z at the middle of the domain
-        rmagx=rmagx,
-        zmagx=zmagx,  # Location of magnetic axis
-        simagx=simagx,  # Poloidal flux at the axis (Weber / rad)
-        sibdry=sibdry,  # Poloidal flux at plasma boundary (Weber / rad)
-        cpasma=cpasma,  #
-        psi=f,  # Poloidal flux in Weber/rad on grid points
-        fpol=fpol,  # Poloidal current function on uniform flux grid
-        pres=pres,  # Plasma pressure in nt/m^2 on uniform flux grid
-        qpsi=qpsi,  # q values on uniform flux grid
-        nbdry=nbdry,
-        rbdry=rbdry,
-        zbdry=zbdry,  # Plasma boundary
+        rmagx=data["rmagx"],
+        zmagx=data["zmagx"],  # Location of magnetic axis
+        simagx=data["simagx"],  # Poloidal flux at the axis (Weber / rad)
+        sibdry=data["sibdry"],  # Poloidal flux at plasma boundary (Weber / rad)
+        cpasma=data["current"],  #
+        psi=data["psirz"].T,  # Poloidal flux in Weber/rad on grid points
+        fpol=data["fpol"],  # Poloidal current function on uniform flux grid
+        pres=data["pres"],  # Plasma pressure in nt/m^2 on uniform flux grid
+        qpsi=data["qpsi"],  # q values on uniform flux grid
+        nbdry=data["nbbbs"],
+        rbdry=data["rbbbs"],
+        zbdry=data["zbbbs"],  # Plasma boundary
         nlim=nlim,
         xlim=xlim,
         ylim=ylim,
-    )  # Wall boundary
+    )

From 76ff80b6e34e72451a87625a9f0758a5169808e7 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Fri, 2 Aug 2024 17:30:16 +0100
Subject: [PATCH 15/17] Move `read_geqdsk` into `geqdsk.py`

---
 src/boututils/geqdsk.py      | 60 +++++++++++++++++++++++++++++++++++
 src/boututils/read_geqdsk.py | 61 ------------------------------------
 2 files changed, 60 insertions(+), 61 deletions(-)
 delete mode 100644 src/boututils/read_geqdsk.py

diff --git a/src/boututils/geqdsk.py b/src/boututils/geqdsk.py
index 969fbd6d0..e06e49bca 100644
--- a/src/boututils/geqdsk.py
+++ b/src/boututils/geqdsk.py
@@ -8,6 +8,8 @@
 import freeqdsk
 import numpy
 
+from .bunch import Bunch
+
 
 class Geqdsk:
     def __init__(self, filename: str):
@@ -125,6 +127,64 @@ def getDescriptor(self, varname):
         return self.data[varname.lower()][1]
 
 
+def read_geqdsk(filename: str):
+
+    data = Geqdsk(filename)
+
+    nxefit = data["nw"]
+    nyefit = data["nh"]
+    xdim = data["rdim"]
+    zdim = data["zdim"]
+    rgrid1 = data["rleft"]
+    zmid = data["zmid"]
+
+    nlim = data["limitr"]
+
+    if nlim != 0:
+        xlim = data["rlim"]
+        ylim = data["zlim"]
+    else:
+        xlim = [0]
+        ylim = [0]
+
+    # Reconstruct the (R,Z) mesh
+    r = numpy.zeros((nxefit, nyefit), numpy.float64)
+    z = numpy.zeros((nxefit, nyefit), numpy.float64)
+
+    for i in range(0, nxefit):
+        for j in range(0, nyefit):
+            r[i, j] = rgrid1 + xdim * i / (nxefit - 1)
+            z[i, j] = (zmid - 0.5 * zdim) + zdim * j / (nyefit - 1)
+
+    return Bunch(
+        nx=nxefit,
+        ny=nyefit,  # Number of horizontal and vertical points
+        r=r,
+        z=z,  # Location of the grid-points
+        xdim=xdim,
+        zdim=zdim,  # Size of the domain in meters
+        rcentr=data["rcentr"],
+        bcentr=data["bcentr"],  # Reference vacuum toroidal field (m, T)
+        rgrid1=rgrid1,  # R of left side of domain
+        zmid=zmid,  # Z at the middle of the domain
+        rmagx=data["rmagx"],
+        zmagx=data["zmagx"],  # Location of magnetic axis
+        simagx=data["simagx"],  # Poloidal flux at the axis (Weber / rad)
+        sibdry=data["sibdry"],  # Poloidal flux at plasma boundary (Weber / rad)
+        cpasma=data["current"],  #
+        psi=data["psirz"].T,  # Poloidal flux in Weber/rad on grid points
+        fpol=data["fpol"],  # Poloidal current function on uniform flux grid
+        pres=data["pres"],  # Plasma pressure in nt/m^2 on uniform flux grid
+        qpsi=data["qpsi"],  # q values on uniform flux grid
+        nbdry=data["nbbbs"],
+        rbdry=data["rbbbs"],
+        zbdry=data["zbbbs"],  # Plasma boundary
+        nlim=nlim,
+        xlim=xlim,
+        ylim=ylim,
+    )
+
+
 ################################
 
 
diff --git a/src/boututils/read_geqdsk.py b/src/boututils/read_geqdsk.py
deleted file mode 100644
index 09626672a..000000000
--- a/src/boututils/read_geqdsk.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import numpy
-
-from .bunch import Bunch
-from .geqdsk import Geqdsk
-
-
-def read_geqdsk(filename: str):
-    data = Geqdsk(filename)
-
-    nxefit = data["nw"]
-    nyefit = data["nh"]
-    xdim = data["rdim"]
-    zdim = data["zdim"]
-    rgrid1 = data["rleft"]
-    zmid = data["zmid"]
-
-    nlim = data["limitr"]
-
-    if nlim != 0:
-        xlim = data["rlim"]
-        ylim = data["zlim"]
-    else:
-        xlim = [0]
-        ylim = [0]
-
-    # Reconstruct the (R,Z) mesh
-    r = numpy.zeros((nxefit, nyefit), numpy.float64)
-    z = numpy.zeros((nxefit, nyefit), numpy.float64)
-
-    for i in range(0, nxefit):
-        for j in range(0, nyefit):
-            r[i, j] = rgrid1 + xdim * i / (nxefit - 1)
-            z[i, j] = (zmid - 0.5 * zdim) + zdim * j / (nyefit - 1)
-
-    return Bunch(
-        nx=nxefit,
-        ny=nyefit,  # Number of horizontal and vertical points
-        r=r,
-        z=z,  # Location of the grid-points
-        xdim=xdim,
-        zdim=zdim,  # Size of the domain in meters
-        rcentr=data["rcentr"],
-        bcentr=data["bcentr"],  # Reference vacuum toroidal field (m, T)
-        rgrid1=rgrid1,  # R of left side of domain
-        zmid=zmid,  # Z at the middle of the domain
-        rmagx=data["rmagx"],
-        zmagx=data["zmagx"],  # Location of magnetic axis
-        simagx=data["simagx"],  # Poloidal flux at the axis (Weber / rad)
-        sibdry=data["sibdry"],  # Poloidal flux at plasma boundary (Weber / rad)
-        cpasma=data["current"],  #
-        psi=data["psirz"].T,  # Poloidal flux in Weber/rad on grid points
-        fpol=data["fpol"],  # Poloidal current function on uniform flux grid
-        pres=data["pres"],  # Plasma pressure in nt/m^2 on uniform flux grid
-        qpsi=data["qpsi"],  # q values on uniform flux grid
-        nbdry=data["nbbbs"],
-        rbdry=data["rbbbs"],
-        zbdry=data["zbbbs"],  # Plasma boundary
-        nlim=nlim,
-        xlim=xlim,
-        ylim=ylim,
-    )

From 6f0cb73cfca82afce9ced14b80994a30ce141ae6 Mon Sep 17 00:00:00 2001
From: Peter Hill <peter.hill@york.ac.uk>
Date: Fri, 2 Aug 2024 17:30:50 +0100
Subject: [PATCH 16/17] Add deprecation warnings to geqdsk readers

---
 src/boututils/geqdsk.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/boututils/geqdsk.py b/src/boututils/geqdsk.py
index e06e49bca..f55ce3a7f 100644
--- a/src/boututils/geqdsk.py
+++ b/src/boututils/geqdsk.py
@@ -7,6 +7,7 @@
 
 import freeqdsk
 import numpy
+import warnings
 
 from .bunch import Bunch
 
@@ -16,6 +17,11 @@ def __init__(self, filename: str):
         """
         Constructor
         """
+        warnings.warn(
+            "`boututils.geqdsk.Geqdsk` is deprecated, use `freeqdsk.geqdsk.read` directly",
+            DeprecationWarning,
+        )
+
         self.data = {}
 
         self.openFile(filename)
@@ -128,6 +134,10 @@ def getDescriptor(self, varname):
 
 
 def read_geqdsk(filename: str):
+    warnings.warn(
+        "`boututils.geqdks.read_geqdsk` is deprecated, use `freeqdsk.geqdsk.read` directly",
+        DeprecationWarning,
+    )
 
     data = Geqdsk(filename)
 

From fcac277c637efcb73d9991b0ea30c53ebacd0dcf Mon Sep 17 00:00:00 2001
From: ZedThree <ZedThree@users.noreply.github.com>
Date: Fri, 2 Aug 2024 16:34:14 +0000
Subject: [PATCH 17/17] [skip ci] Apply black/isort changes

---
 src/boututils/geqdsk.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/boututils/geqdsk.py b/src/boututils/geqdsk.py
index f55ce3a7f..5d808e488 100644
--- a/src/boututils/geqdsk.py
+++ b/src/boututils/geqdsk.py
@@ -5,9 +5,10 @@
 https://fusion.gat.com/conferences/snowmass/working/mfe/physics/p3/equilibria/g_eqdsk_s.pdf
 """
 
+import warnings
+
 import freeqdsk
 import numpy
-import warnings
 
 from .bunch import Bunch