Introduce ecoli.library.xarray_emitter

.gitignore

@@ -69,4 +69,4 @@ trace-*
 test_sherlock/
 
 # SMS API #
-.hpc_env
+.hpc_env

configs/test_configs/test_xarray_emitter.json

@@ -0,0 +1,129 @@
+{
+  "experiment_id": "test_xarray_emitter",
+  "fixed_media": "minimal",
+  "suffix_time": false,
+  "max_duration": 10.0,
+  "fail_at_max_duration": false,
+  "generations": 1,
+  "n_init_sims": 1,
+  "log_updates": true,
+  "emitter": "xarray",
+  "emitter_arg": {
+    "out_dir": "out/tests",
+    "debug": false,
+    "transducer": {
+      "predicate": [
+        [
+          {"subsample": {"interval": 1}},
+          {"fixed": {"steps": [0]}}
+        ]
+      ],
+      "buffer": {
+        "size": 3
+      }
+    },
+    "writer": {
+      "threaded": true,
+      "buffers_per_chunk": 2,
+      "backend": "zarr",
+      "backend_config": {
+        "format": 3,
+        "async.concurrency": 3,
+        "threading.max_workers": 3
+      }
+    },
+    "view": [
+      {
+        "root": [],
+        "variables": {
+          "bulk": [{
+            "path": "bulk/bulk_molecule",
+            "dtype": "<i8",
+            "codecs": {
+              "filters_v2": [],
+              "filters_v3": [],
+              "compressors_v2": [{
+                "id": "lzma", "format": 3, "check": -1, "preset": null,
+                "filters": [{"id": 3, "dist": 8}, {"id": 33, "preset": 5}]
+              }],
+              "compressors_v3": [{
+                "name": "numcodecs.lzma",
+                "configuration": {
+                  "format": 3,
+                  "filters": [{"id": 3, "dist": 8}, {"id": 33, "preset": 5}]
+                }
+              }]
+            }
+          }]
+        }
+      },
+      {
+        "root": ["listeners", "mass"],
+        "metadata": false,
+        "variables": {
+          "cell_mass": [{
+            "path": "gauges/cell_mass",
+            "unit": "[fg]",
+            "dtype": "<f4"
+          }],
+          "dry_mass": [{
+            "path": "gauges/dry_mass",
+            "unit": "[fg]",
+            "dtype": "<f4"
+          }]
+        }
+      },
+      {
+        "root": ["listeners"],
+        "variables": {
+          "fba_results": {
+            "reaction_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn_global_listener",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }]
+          }
+        }
+      },
+      {
+        "root": ["log_update", "ecoli-metabolism", "listeners"],
+        "variables": {
+          "fba_results": {
+            "coefficient": [{
+              "path": "gauges/mass_per_vol",
+              "unit": "[g/L]",
+              "dtype": "<f4"
+            }],
+            "reaction_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }],
+            "external_exchange_fluxes": [{
+              "path": "metabolism/fluxes/exchange/molecule",
+              "unit": "[mmol/g.h]",
+              "dtype": "<f4"
+            }]
+          },
+          "enzyme_kinetics": {
+            "counts_to_molar": [{
+              "path": "gauges/conc_per_count",
+              "unit": "[mmol/L]",
+              "dtype": "<f4"
+            }],
+            "actual_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn_constrained",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }],
+            "target_fluxes": [{
+              "path": "metabolism/fluxes/internal/rxn_target",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }]
+          }
+        }
+      }
+    ]
+  }
+}

configs/test_configs/test_xarray_workflow.json

@@ -0,0 +1,47 @@
+{
+  "experiment_id": "test_xarray_workflow",
+  "sim_data_path": "out/kb/simData.cPickle",
+  "suffix_time": false,
+  "fail_at_max_duration": true,
+  "generations": 6,
+  "n_init_sims": 10,
+  "skip_baseline": false,
+  "variants": {
+    "condition": {"condition": {"value": ["with_aa", "acetate"]}
+    }
+  },
+  "emitter": "xarray",
+  "emitter_arg": {
+    "out_dir": "out/experiments",
+    "transducer": {
+      "predicate": [
+        [{"subsample": {"interval": 3}}]
+      ],
+      "buffer": {"size": 200}
+    },
+    "writer": {
+      "threaded": true,
+      "buffers_per_chunk": 1,
+      "backend": "zarr",
+      "backend_config": {
+        "format": 3,
+        "async.concurrency": 3,
+        "threading.max_workers": 3
+      }
+    },
+    "view": [
+      {
+        "root": ["listeners"],
+        "variables": {
+          "fba_results": {
+            "reaction_fluxes": [{
+              "path": "metabolic_fluxes",
+              "unit": "[mmol/L.s]",
+              "dtype": "<f4"
+            }]
+          }
+        }
+      }
+    ]
+  }
+}

doc/apidoc_templates/module.rst_t

@@ -1,7 +1,5 @@
 {{- "``{}``".format(basename) | heading }}
 .. automodule:: {{ qualname }}
-   :private-members:
-   :exclude-members: _abc_impl 
 {%- for option in automodule_options %}
    :{{ option }}:
 {%- endfor %}

doc/composites.rst

@@ -72,6 +72,9 @@ their corresponding topologies to the final composite model.
 - ``generations``: Adds :py:class:`~ecoli.processes.cell_division.StopAfterDivision`
   when ``True`` but only if ``divide`` is ``True``
 
+
+.. _log_updates:
+
 -----------
 Log Updates
 -----------
@@ -90,8 +93,10 @@ The analysis plots located in :py:mod:`~ecoli.analysis.single.blame` can be used
 to visualize these updates.
 
 .. warning::
-    This feature should only be turned for debugging purposes and
-    only when using the in-memory emitter (see :ref:`ram_emitter`).
+    This feature should only be turned on for debugging purposes, and only when
+    using the :ref:`in-memory emitter <ram_emitter>` or the
+    :py:mod:`.xarray_emitter`.
+
 
 -------------
 Initial State

doc/conf.py

@@ -86,6 +86,21 @@
     # Silence warning in ecoli.processes.environment.field_timeline.FieldTimeline
     ("py:class", "vivarium.processes.timeline.TimelineProcess"),
     ("py:class", "concurrent.futures._base.Future"),
+    # Type annotations using library internals
+    ("py:class", "concurrent.futures._base.Executor"),
+    ("py:class", "unittest.mock._patch"),
+    ("py:class", "xarray.backends.common.ArrayWriter"),
+    ("py:class", "xarray.core.treenode.NodePath"),
+    ("py:class", "zarr.core.group.ConsolidatedMetadata"),
+    ("py:class", "zarr.core._tree.TreeRepr"),
+    # Sphinx does not recognize type parameters in generic classes
+    ("py:class", "ArrT"),
+    ("py:class", "NodeT"),
+    ("py:class", "StoreT"),
+    # Sphinx does not recognize type aliases
+    ("py:type", "VariableEncoding"),
+    ("py:class", "VariableEncoding"),
+    ("py:class", "AnyAsyncArray"),
 ]
 
 
@@ -115,15 +130,17 @@
 # -- sphinx.ext.intersphinx options --
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
-    "vivarium": (
-        "https://vivarium-core.readthedocs.io/en/latest/",
-        None,
-    ),
+    "pytest": ("https://docs.pytest.org/en/latest", None),
+    "vivarium": ("https://vivarium-core.readthedocs.io/en/latest/", None),
     "numpy": ("https://numpy.org/doc/stable", None),
+    "xarray": ("https://docs.xarray.dev/en/latest", None),
+    "zarr": ("https://zarr.readthedocs.io/en/latest", None),
     "matplotlib": ("https://matplotlib.org/stable/", None),
     "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None),
     "polars": ("https://docs.pola.rs/api/python/stable", None),
     "sympy": ("https://docs.sympy.org/latest", None),
+    "pint": ("https://pint.readthedocs.io/en/stable", None),
+    "unum": ("https://unum.readthedocs.io/en/stable", None),
 }
 
 
@@ -143,8 +160,24 @@
 ]
 # Move typehints from signature into description
 autodoc_typehints = "description"
-# Concatenate class and __init__ docstrings
-autoclass_content = "both"
+# Only use the class’s docstring. __init__ docstrings are now listed separately.
+autoclass_content = "class"
+# Default options for all autodoc directives.
+autodoc_default_options = {
+    "member-order": "bysource",
+    "private-members": True,
+    "special-members": (
+        # object
+        "__init__,  __del__, __call__"
+    ),
+    "exclude-members": (
+        # abc.ABC
+        "_abc_impl, "
+        # enum.Flag
+        "_flag_mask_, _singles_mask_, _all_bits_, _boundary_, _inverted_, "
+        "_generate_next_value_"
+    )
+}
 # Remove domain objects (e.g. functions, classes, attributes) from
 # table of contents
 toc_object_entries = False

doc/experiments.rst

@@ -396,6 +396,9 @@ Here are some general rules to remember when writing your own JSON config files:
     without dividing, this results in a more informative error message instead
     of a Nextflow error about missing daughter cell states.
 
+
+.. _experiment_output:
+
 ------
 Output
 ------

doc/stores.rst

@@ -368,6 +368,8 @@ any number of attributes for all active (``_entryState`` is 1) unique molecules
 of a given type (e.g. RNA, active RNAP, etc.).
 
 
+.. _listeners:
+
 ---------
 Listeners
 ---------

doc/workflows.rst

@@ -330,6 +330,7 @@ it has access to.
   folder, you can just create stub files in the appropriate folders
   that simply import the ``plot`` function from a primary analysis script.
 
+
 .. _analysis_config:
 
 Configuration
@@ -502,6 +503,9 @@ Refer to :ref:`/output.rst` for more information about how
 to use DuckDB to read and analyze simulation output inside
 analysis scripts.
 
+
+.. _workflows:
+
 ---------
 Workflows
 ---------
@@ -700,6 +704,7 @@ is a list workflow behaviors enabled in our model to handle unexpected errors.
   depends on generation 6, :py:mod:`runscripts.create_variants` depends on
   :py:mod:`runscripts.parca`, etc).
 
+
 .. _output:
 
 ------

ecoli/__init__.py

@@ -6,6 +6,7 @@
 )
 
 from ecoli.library.parquet_emitter import ParquetEmitter
+from ecoli.library.xarray_emitter.emitter import XarrayEmitter
 from ecoli.library.schema import (
     divide_binomial,
     divide_bulk,
@@ -39,6 +40,7 @@
 faulthandler.enable()
 
 emitter_registry.register("parquet", ParquetEmitter)
+emitter_registry.register("xarray", XarrayEmitter)
 
 # register :term:`updaters`
 inverse_updater_registry.register("accumulate", inverse_update_accumulate)