iddata refactoring

CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.0.0]
+
+### Added
+- `features.py` module with composable feature engineering classes: `Feature` (abstract base), `OneHotEncodingFeature`, `HolidayFeature`, `TaylorFeature`, `RollingMeanFeature`, `LagFeature`, `HorizonTargetFeature`, `LevelFeatureFilter`, `DirectionalWaveFeature`, `FeaturePipeline`
+- `transforms.py` module with `Transform` (abstract base), `FourthRootTransform`, `IdentityTransform`, `CenterScaleTransform`, `ComposedTransform`
+- `model.py` module with `IDModel` abstract base class shared by `GBQRModel` and `SARIXModel`
+- `SourceType` exported from `idmodels` top-level (sourced from `iddata.enums`)
+- Unit tests for `HolidayFeature`, `TaylorFeature`, `RollingMeanFeature`, `CenterScaleTransform`, and `FeaturePipeline` edge cases
+- `tests/integration/conftest.py` with shared `make_run_config` pytest fixture, replacing duplicate `create_test_gbqr_run_config` and `create_test_sarix_run_config` helpers
+
+### Changed
+- **Breaking**: `DataSource` enum removed from `idmodels`; replace with `SourceType` from `iddata.enums`
+- **Breaking**: `model_config.sources` now expects `list[SourceType]` instead of `list[DataSource]`
+- **Breaking**: `preprocess.py` replaced by `features.py`; feature engineering is now class-based via `FeaturePipeline`
+- `GBQRModel` and `SARIXModel` refactored to extend `IDModel` base class and use `FeaturePipeline`
+- `Disease` and `SourceType` enums now sourced directly from `iddata.enums`
+- Updated `iddata` dependency to latest commit
+- Ruff isort config: pinned `idmodels` as `known-first-party` for consistent cross-environment linting
+
+### Removed
+- `DataSource` enum (use `SourceType` from `iddata.enums`)
+- `preprocess.py` (superseded by `features.py`)
+
 ## [1.3.1]
 
 ### Changed
@@ -102,7 +125,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Updated to latest iddata API
 
-[Unreleased]: https://github.com/reichlab/idmodels/compare/v1.3.0...HEAD
+[Unreleased]: https://github.com/reichlab/idmodels/compare/v2.0.0...HEAD
+[2.0.0]: https://github.com/reichlab/idmodels/compare/v1.3.1...v2.0.0
 [1.3.0]: https://github.com/reichlab/idmodels/compare/v1.1.0...v1.3.0
 [1.1.0]: https://github.com/reichlab/idmodels/compare/v1.0.0...v1.1.0
 [1.0.0]: https://github.com/reichlab/idmodels/compare/v0.1.0...v1.0.0

pyproject.toml

@@ -11,7 +11,7 @@ classifiers = [
 dynamic = ["version"]
 
 dependencies = [
-    "iddata @ git+https://github.com/reichlab/iddata",
+    "iddata @ git+https://github.com/reichlab/iddata@ba1be8a957a6012c83ceff670e6e38b75b6f219b",
     "lightgbm",
     "numpy",
     "pandas~=2.0",  # pandas 3.0 breaks compatibility; remove cap once validated
@@ -52,6 +52,9 @@ idmodels = ["data/*.csv", "data/*.md"]
 line-length = 120
 lint.extend-select = ["I", "Q"]
 
+[tool.ruff.lint.isort]
+known-first-party = ["idmodels"]
+
 [tool.ruff.lint.flake8-quotes]
 inline-quotes = "double"
 

requirements/requirements-dev.txt

@@ -38,7 +38,7 @@ frozenlist==1.5.0
     #   aiosignal
 fsspec==2024.10.0
     # via s3fs
-iddata @ git+https://github.com/reichlab/iddata@5a7e74d7823d39b8a8ef6334c5191e440bc669d8
+iddata @ git+https://github.com/reichlab/iddata@ba1be8a957a6012c83ceff670e6e38b75b6f219b
     # via idmodels (pyproject.toml)
 identify==2.6.1
     # via pre-commit

requirements/requirements.txt

@@ -30,7 +30,7 @@ frozenlist==1.5.0
     #   aiosignal
 fsspec==2024.10.0
     # via s3fs
-iddata @ git+https://github.com/reichlab/iddata@5a7e74d7823d39b8a8ef6334c5191e440bc669d8
+iddata @ git+https://github.com/reichlab/iddata@ba1be8a957a6012c83ceff670e6e38b75b6f219b
     # via idmodels (pyproject.toml)
 idna==3.10
     # via yarl

src/app.py

@@ -0,0 +1,26 @@
+import pickle
+from pathlib import Path
+
+from idmodels.sarix import SARIXModel
+
+
+def main():
+    pkl_dir = Path("/Users/cornell/IdeaProjects/operational-models/covid_gbqr/")
+    # pkl_dir = Path('/Users/cornell/IdeaProjects/operational-models/covid_ar6_pooled/')
+    pkl_mc_rc_pairs = [
+        # ('2025-08-02-model_config.pkl', '2025-08-02-run_config.pkl'),  # works -> 2025-08-02-UMass-gbqr.csv
+        ("2025-08-09-model_config.pkl", "2025-08-09-run_config.pkl"),  # fails -> 2025-08-09-UMass-gbqr.csv
+    ]
+    for model_config_file_name, run_config_file_name in pkl_mc_rc_pairs:
+        with open(pkl_dir / model_config_file_name, "rb") as mc_fp, open(pkl_dir / run_config_file_name, "rb") as rc_fp:
+            model_config = pickle.load(mc_fp)
+            run_config = pickle.load(rc_fp)
+        print("*", model_config_file_name, ",", run_config_file_name)
+        print("yy", model_config)
+        model_config.x = []
+        model = SARIXModel(model_config)
+        model.run(run_config)
+
+
+if __name__ == "__main__":
+    main()

src/idmodels/__init__.py

@@ -1,17 +1,28 @@
+from iddata.enums import Disease, SourceType
+
 from idmodels.config import (
-                             DataSource,
-                             Disease,
-                             GBQRModelConfig,
-                             PoolingStrategy,
-                             PowerTransform,
-                             RunConfig,
-                             SARIXFourierModelConfig,
-                             SARIXModelConfig,
+    GBQRModelConfig,
+    PoolingStrategy,
+    PowerTransform,
+    RunConfig,
+    SARIXFourierModelConfig,
+    SARIXModelConfig,
 )
 from idmodels.gbqr import GBQRModel
 from idmodels.sarix import SARIXFourierModel, SARIXModel
 
-__all__ = ["DataSource", "Disease", "GBQRModel", "GBQRModelConfig", "PoolingStrategy", "PowerTransform", "RunConfig",
-           "SARIXFourierModel", "SARIXFourierModelConfig", "SARIXModel", "SARIXModelConfig"]
+__all__ = [
+    "Disease",
+    "GBQRModel",
+    "GBQRModelConfig",
+    "PoolingStrategy",
+    "PowerTransform",
+    "RunConfig",
+    "SARIXFourierModel",
+    "SARIXFourierModelConfig",
+    "SARIXModel",
+    "SARIXModelConfig",
+    "SourceType",
+]
 
-__version__ = "1.3.1"
+__version__ = "2.0.0"

src/idmodels/config.py

@@ -4,18 +4,10 @@
 from enum import Enum
 from pathlib import Path
 
-
-class DataSource(str, Enum):
-    NHSN = "nhsn"
-    NSSP = "nssp"
-    FLUSURVNET = "flusurvnet"
-    ILINET = "ilinet"
-
-
-class Disease(str, Enum):
-    FLU = "flu"
-    COVID = "covid"
-    RSV = "rsv"
+from iddata.enums import (
+    Disease,  # used internally for RunConfig.disease; import from iddata.enums directly in callers
+    SourceType,  # re-exported for callers: from idmodels.config import SourceType
+)
 
 
 class PowerTransform(str, Enum):
@@ -30,30 +22,22 @@ class PoolingStrategy(str, Enum):
 
 @dataclass
 class ModelConfig(ABC):
-    """
-    Abstract base for model configuration.
-
-    Holds settings that describe *what* model to run and how it processes data (sources, transforms, pooling).
-    Not instantiated directly - use :class:`SARIXModelConfig` or :class:`GBQRModelConfig`.
-    """
+    """Abstract base for model configuration."""
 
     model_name: str
-    sources: list[DataSource]
+    sources: list[SourceType]
     fit_locations_separately: bool
     power_transform: PowerTransform
 
+
     def __post_init__(self):
         if type(self) is ModelConfig:
             raise TypeError("ModelConfig is abstract - use SARIXModelConfig or GBQRModelConfig")
 
 
 @dataclass
 class RunConfig:
-    """
-    Run configuration.
-
-    Holds settings that describe a single execution: which disease, which locations, output paths, quantile levels, etc.
-    """
+    """Run configuration: disease, locations, output paths, quantile levels."""
 
     disease: Disease
     ref_date: datetime.date
@@ -95,7 +79,6 @@ class GBQRModelConfig(ModelConfig):
     reporting_adj: bool = False
     save_feat_importance: bool = False
 
-    # directional wave features (disabled by default)
     use_directional_waves: bool = False
     wave_directions: list[str] = field(default_factory=lambda: ["N", "NE", "E", "SE", "S", "SW", "W", "NW"])
     wave_temporal_lags: list[int] = field(default_factory=lambda: [1, 2])

src/idmodels/constants.py

@@ -0,0 +1,4 @@
+POWER_TRANSFORM_OFFSET: float = 0.01
+
+IN_SEASON_WEEK_MIN: int = 10
+IN_SEASON_WEEK_MAX: int = 45