diff --git a/benchmarks/cli/introspect.py b/benchmarks/cli/introspect.py index 41e5dc3d..1a78022f 100644 --- a/benchmarks/cli/introspect.py +++ b/benchmarks/cli/introspect.py @@ -94,7 +94,7 @@ def _row(label: str, value: object) -> None: _row("features:", sorted(spec.features)) _row("phases:", sorted(spec.phases)) _row("quick:", spec.quick_subset) - _row("long_threshold:", spec.long_threshold) + _row("long:", ", ".join(str(s) for s in spec.long_sizes) or "—") if spec.requires: _row("requires:", list(spec.requires)) return @@ -106,7 +106,7 @@ def _row(label: str, value: object) -> None: _row("description:", pattern.description) _row("phases:", sorted(pattern.phases)) _row("quick:", pattern.quick_subset) - _row("long_threshold:", pattern.long_threshold) + _row("long:", ", ".join(str(s) for s in pattern.long_sizes) or "—") if pattern.requires: _row("requires:", list(pattern.requires)) return diff --git a/benchmarks/cli/run.py b/benchmarks/cli/run.py index 4e060436..6e42b990 100644 --- a/benchmarks/cli/run.py +++ b/benchmarks/cli/run.py @@ -71,7 +71,7 @@ def run( bool, typer.Option( "--long", - help="Include the slowest sizes (above each spec's long_threshold).", + help="Include the slowest sizes (each spec's long_sizes).", ), ] = False, phase: Annotated[ @@ -136,7 +136,7 @@ def run( them sequentially. Results print to the terminal; pass ``--json PATH`` to also save a snapshot (one rule for both metrics). - Without ``--quick``/``--long``, sizes above each spec's ``long_threshold`` + Without ``--quick``/``--long``, each spec's ``long_sizes`` (the heaviest) are skipped — keeps the wall-clock manageable. ``--size``/``--severity`` pin exact values on either axis. diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index b78390d7..eda9ed6c 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -29,7 +29,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: action="store_true", default=False, help=( - "Include the slowest sizes (above each spec's long_threshold). " + "Include the slowest sizes (each spec's long_sizes). " "Default runs skip them." ), ) @@ -86,7 +86,7 @@ def maybe_skip(request: pytest.FixtureRequest, spec: BenchSpec, size: int) -> No - ``--size N`` / ``--severity S`` → run only the listed values for that axis (models read ``--size``, patterns ``--severity``); overrides tiers. - ``--quick`` → only ``spec.quick_subset`` - - default (no flag) → skip ``size > long_threshold`` + - default (no flag) → skip sizes in ``spec.long_sizes`` - ``--long`` → no size cap A manual axis flag wins over ``--quick``/``--long``; ``--quick`` in turn diff --git a/benchmarks/models/basic.py b/benchmarks/models/basic.py index a41f75ae..8d20c5d1 100644 --- a/benchmarks/models/basic.py +++ b/benchmarks/models/basic.py @@ -6,6 +6,8 @@ from benchmarks.registry import CONTINUOUS, ModelSpec, register SIZES = (10, 50, 100, 250, 500, 1000, 1600) +QUICK_SIZES = (10, 250) +LONG_SIZES = (1000, 1600) def build_basic(n: int) -> linopy.Model: @@ -24,7 +26,8 @@ def build_basic(n: int) -> linopy.Model: name="basic", build=build_basic, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS}), - long_threshold=500, ) ) diff --git a/benchmarks/models/expression_arithmetic.py b/benchmarks/models/expression_arithmetic.py index 0f687f05..3616718a 100644 --- a/benchmarks/models/expression_arithmetic.py +++ b/benchmarks/models/expression_arithmetic.py @@ -8,6 +8,8 @@ from benchmarks.registry import CONTINUOUS, ModelSpec, register SIZES = (10, 50, 100, 250, 500, 1000) +QUICK_SIZES = (10, 250) +LONG_SIZES = (1000,) def build_expression_arithmetic(n: int) -> linopy.Model: @@ -36,7 +38,8 @@ def build_expression_arithmetic(n: int) -> linopy.Model: name="expression_arithmetic", build=build_expression_arithmetic, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS}), - long_threshold=500, ) ) diff --git a/benchmarks/models/knapsack.py b/benchmarks/models/knapsack.py index 33dc1ca8..b82b3c75 100644 --- a/benchmarks/models/knapsack.py +++ b/benchmarks/models/knapsack.py @@ -8,6 +8,8 @@ from benchmarks.registry import BINARY, DEFAULT_PHASES, ModelSpec, register SIZES = (100, 1_000, 10_000, 100_000, 1_000_000) +QUICK_SIZES = (100, 10_000) +LONG_SIZES = (100_000, 1_000_000) def build_knapsack(n: int) -> linopy.Model: @@ -29,8 +31,9 @@ def build_knapsack(n: int) -> linopy.Model: name="knapsack", build=build_knapsack, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({BINARY}), phases=DEFAULT_PHASES, # HiGHS handles binary; matrices handles MILP - long_threshold=10_000, ) ) diff --git a/benchmarks/models/masked.py b/benchmarks/models/masked.py index ec024d8b..b602240b 100644 --- a/benchmarks/models/masked.py +++ b/benchmarks/models/masked.py @@ -35,6 +35,8 @@ ) SIZES = (10, 50, 100, 500, 1000) +QUICK_SIZES = (10, 100) +LONG_SIZES = (1000,) def build_masked(n: int) -> linopy.Model: @@ -83,8 +85,9 @@ def build_masked(n: int) -> linopy.Model: name="masked", build=build_masked, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS, MASKED}), phases=DEFAULT_PHASES, - long_threshold=500, ) ) diff --git a/benchmarks/models/milp.py b/benchmarks/models/milp.py index 98fe9f99..02f11341 100644 --- a/benchmarks/models/milp.py +++ b/benchmarks/models/milp.py @@ -31,6 +31,8 @@ ) SIZES = (10, 25, 50, 100, 200) +QUICK_SIZES = (10, 50) +LONG_SIZES = (200,) def build_milp(n: int) -> linopy.Model: @@ -72,8 +74,9 @@ def build_milp(n: int) -> linopy.Model: name="milp", build=build_milp, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({INTEGER, CONTINUOUS}), phases=DEFAULT_PHASES, - long_threshold=100, ) ) diff --git a/benchmarks/models/piecewise.py b/benchmarks/models/piecewise.py index 069d135b..a15191ae 100644 --- a/benchmarks/models/piecewise.py +++ b/benchmarks/models/piecewise.py @@ -32,6 +32,8 @@ ) SIZES = (10, 100, 1_000, 5_000) +QUICK_SIZES = (10, 1_000) +LONG_SIZES = (5_000,) _API_AVAILABLE = hasattr(linopy.Model, "add_piecewise_formulation") and hasattr( linopy, "EvolvingAPIWarning" @@ -81,12 +83,13 @@ def build_piecewise(n_gens: int) -> linopy.Model: name="piecewise", build=build_piecewise, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS, PIECEWISE}), # Monotonic breakpoints + ``method="auto"`` → incremental # reformulation (pure MILP with binaries), which every supported # solver handles. phases=DEFAULT_PHASES, - long_threshold=1_000, ) ) else: diff --git a/benchmarks/models/pypsa_scigrid.py b/benchmarks/models/pypsa_scigrid.py index 30641897..2e837dad 100644 --- a/benchmarks/models/pypsa_scigrid.py +++ b/benchmarks/models/pypsa_scigrid.py @@ -10,6 +10,8 @@ import linopy SIZES = (10, 50, 100, 200) +QUICK_SIZES = () # out of --quick entirely (PyPSA import dominates the smoke) +LONG_SIZES = (100, 200) # only the bigger networks under --long def build_pypsa_scigrid(snapshots: int = 100) -> linopy.Model: @@ -31,12 +33,9 @@ def build_pypsa_scigrid(snapshots: int = 100) -> linopy.Model: name="pypsa_scigrid", build=build_pypsa_scigrid, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS}), - # quick_sizes=() keeps pypsa_scigrid out of --quick entirely — PyPSA - # import + example loading dominates the smoke wall-clock otherwise. - # It still runs in default and --long modes. - quick_sizes=(), - long_threshold=50, requires=("pypsa",), ) ) diff --git a/benchmarks/models/qp.py b/benchmarks/models/qp.py index 62b2002f..6b5e313c 100644 --- a/benchmarks/models/qp.py +++ b/benchmarks/models/qp.py @@ -31,6 +31,8 @@ ) SIZES = (10, 100, 1_000, 5_000, 20_000) +QUICK_SIZES = (10, 1_000) +LONG_SIZES = (5_000, 20_000) def build_qp(n_assets: int) -> linopy.Model: @@ -58,8 +60,9 @@ def build_qp(n_assets: int) -> linopy.Model: name="qp", build=build_qp, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS, QUADRATIC}), phases=DEFAULT_PHASES, - long_threshold=1_000, ) ) diff --git a/benchmarks/models/sos.py b/benchmarks/models/sos.py index 163b8763..0d18a83d 100644 --- a/benchmarks/models/sos.py +++ b/benchmarks/models/sos.py @@ -40,6 +40,8 @@ ) SIZES = (10, 100, 1_000, 10_000) +QUICK_SIZES = (10, 1_000) +LONG_SIZES = (10_000,) _N_MODES = 5 _API_AVAILABLE = hasattr(linopy.Model, "add_sos_constraints") @@ -78,6 +80,8 @@ def build_sos(n_gens: int) -> linopy.Model: name="sos", build=build_sos, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS, SOS}), # HiGHS / Mosek lack native SOS in linopy — would need # ``reformulate_sos=True``, which mutates the model and defeats @@ -93,7 +97,6 @@ def build_sos(n_gens: int) -> linopy.Model: TO_XPRESS, } ), - long_threshold=1_000, ) ) else: diff --git a/benchmarks/models/sparse_network.py b/benchmarks/models/sparse_network.py index e213a03b..57fc4bd5 100644 --- a/benchmarks/models/sparse_network.py +++ b/benchmarks/models/sparse_network.py @@ -10,6 +10,8 @@ from benchmarks.registry import CONTINUOUS, ModelSpec, register SIZES = (10, 50, 100, 250, 500, 1000) +QUICK_SIZES = (10, 250) +LONG_SIZES = (1000,) def build_sparse_network(n_buses: int) -> linopy.Model: @@ -56,7 +58,8 @@ def build_sparse_network(n_buses: int) -> linopy.Model: name="sparse_network", build=build_sparse_network, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS}), - long_threshold=500, ) ) diff --git a/benchmarks/models/storage.py b/benchmarks/models/storage.py index 8d76ec69..05f0faab 100644 --- a/benchmarks/models/storage.py +++ b/benchmarks/models/storage.py @@ -20,6 +20,8 @@ from benchmarks.registry import CONTINUOUS, ModelSpec, register SIZES = (10, 50, 100, 250, 500, 1000) +QUICK_SIZES = (10, 250) +LONG_SIZES = (1000,) N_TIME = 168 DECAY = 0.99 ETA = 0.95 @@ -49,8 +51,9 @@ def build_storage(n_storage: int) -> linopy.Model: name="storage", build=build_storage, sizes=SIZES, + quick_sizes=QUICK_SIZES, + long_sizes=LONG_SIZES, features=frozenset({CONTINUOUS}), - long_threshold=500, description="storage SoC recursion via .shift() — bidiagonal intertemporal coupling", ) ) diff --git a/benchmarks/registry.py b/benchmarks/registry.py index 31e74d2f..f3d5f1ad 100644 --- a/benchmarks/registry.py +++ b/benchmarks/registry.py @@ -8,7 +8,8 @@ - ``sizes`` canonical tuned sizes - ``features`` variable / constraint kinds it uses - ``phases`` applicable phases (to_lp, to_highspy, …) -- ``quick_sizes`` ``--quick`` subset (default: first/mid/last) +- ``quick_sizes`` ``--quick`` subset (per spec) +- ``long_sizes`` heaviest sizes, held back to ``--long`` - ``requires`` modules to ``pytest.importorskip`` :: @@ -103,15 +104,15 @@ class ModelSpec: """ Declarative description of one benchmark model. - Three tiers gate the cost of a default ``pytest benchmarks/`` run: + Three explicit size tiers gate run cost (each a subset, or the whole, of + ``sizes`` — declared per spec, no thresholds): - - ``--quick``: only ``quick_subset`` — an explicit subset (defaults to the - first / middle / last of ``sizes``). The per-PR / CI smoke set. - - default: every size up to ``long_threshold`` (medium-cost regression). - - ``--long``: every size, no cap. + - ``--quick``: only ``quick_sizes`` — the per-PR / CI smoke set. + - default: ``sizes`` minus ``long_sizes`` — medium-cost regression. + - ``--long``: every size in ``sizes``. - ``long_threshold`` defaults to "no cap"; set ``quick_sizes`` to override the - derived quick subset (``()`` opts the spec out of ``--quick`` entirely). + ``quick_sizes=()`` opts a spec out of ``--quick`` entirely; ``long_sizes`` + lists the heaviest sizes held back from the default run. """ name: str @@ -120,7 +121,7 @@ class ModelSpec: features: frozenset[str] = frozenset({CONTINUOUS}) phases: frozenset[str] = DEFAULT_PHASES quick_sizes: tuple[int, ...] | None = None - long_threshold: int = 10**9 + long_sizes: tuple[int, ...] = () requires: tuple[str, ...] = () description: str = "" @@ -137,10 +138,14 @@ def axis(self) -> str: @property def quick_subset(self) -> tuple[int, ...]: """ - Sizes that run under ``--quick`` — the derived first/mid/last, - unless ``quick_sizes`` overrides it (``()`` opts out entirely). + ``--quick`` sizes — ``quick_sizes`` (``()`` opts out); falls back to + first/mid/last of ``sizes`` if unset. """ - return _quick_subset(self.sweep) if self.quick_sizes is None else self.quick_sizes + return ( + self.quick_sizes + if self.quick_sizes is not None + else _quick_subset(self.sweep) + ) def applies_to(self, phase: str) -> bool: return phase in self.phases @@ -165,7 +170,7 @@ def _repr_html_(self) -> str: ("sizes", ", ".join(str(s) for s in self.sizes)), ("phases", ", ".join(sorted(self.phases))), ("quick", ", ".join(str(s) for s in self.quick_subset) or "—"), - ("long_threshold", self.long_threshold), + ("long", ", ".join(str(s) for s in self.long_sizes) or "—"), ("requires", ", ".join(self.requires) or "—"), ] body = "".join( @@ -247,7 +252,8 @@ def param_ids(params: list[tuple[BenchSpec, int]]) -> list[str]: # --- Patterns --------------------------------------------------------------- -DEFAULT_SEVERITIES: tuple[int, ...] = (0, 25, 50, 75, 100) +DEFAULT_SEVERITIES: tuple[int, ...] = (0, 25, 50, 75, 100) # full sweep / --long +QUICK_SEVERITIES: tuple[int, ...] = (0, 50, 100) # --quick (per-PR) class BenchSpec(Protocol): @@ -272,7 +278,7 @@ def requires(self) -> tuple[str, ...]: ... @property def quick_subset(self) -> tuple[int, ...]: ... @property - def long_threshold(self) -> int: ... + def long_sizes(self) -> tuple[int, ...]: ... @property def build(self) -> Callable[[int], linopy.Model]: ... @property @@ -297,10 +303,10 @@ class PatternSpec: A pattern builds a complete model, so it runs the same ``phases`` as a model by default — the build-vs-export contrast (does the dense-``_term`` bloat reach the matrix / LP file, or collapse?) is the point. The full severity - range (``0, 25, 50, 75, 100``) runs by default; ``--quick`` keeps the - ``quick_subset`` (first/middle/last of ``severities`` — ``(0, 50, 100)``) so - smoke exercises the benign, midpoint *and* worst-case shapes, while the full - sweep keeps the finer resolution. + range ``DEFAULT_SEVERITIES`` runs by default; ``--quick`` keeps + ``QUICK_SEVERITIES`` ``(0, 50, 100)`` — the benign, midpoint *and* worst-case + shapes — while the full sweep keeps the finer resolution. Patterns have no + ``long_sizes`` (every severity runs by default). """ name: str @@ -310,7 +316,7 @@ class PatternSpec: phases: frozenset[str] = DEFAULT_PHASES requires: tuple[str, ...] = () quick_sizes: tuple[int, ...] | None = None - long_threshold: int = 10**9 + long_sizes: tuple[int, ...] = () @property def sweep(self) -> tuple[int, ...]: @@ -323,10 +329,12 @@ def axis(self) -> str: @property def quick_subset(self) -> tuple[int, ...]: """ - Severities that run under ``--quick`` — the derived first/mid/last, - unless ``quick_sizes`` overrides it (``()`` opts out entirely). + ``--quick`` severities — ``quick_sizes`` if set, else + ``QUICK_SEVERITIES`` ``(0, 50, 100)``. """ - return _quick_subset(self.sweep) if self.quick_sizes is None else self.quick_sizes + return ( + self.quick_sizes if self.quick_sizes is not None else QUICK_SEVERITIES + ) def applies_to(self, phase: str) -> bool: return phase in self.phases @@ -403,8 +411,8 @@ def skip_reason( - a manual axis list (``sizes`` for models, ``severities`` for patterns) → run only those values; - ``--quick`` → only ``spec.quick_subset``; - - default → skip ``value > long_threshold``; - - ``--long`` → no cap. + - default → skip values in ``spec.long_sizes`` (the heaviest, held back); + - ``--long`` → everything. """ manual = severities if spec.axis == "severity" else sizes if manual: @@ -413,6 +421,6 @@ def skip_reason( if value not in spec.quick_subset: return f"--quick: skipping {spec.name} {spec.axis}={value}" return None - if not long and value > spec.long_threshold: + if not long and value in spec.long_sizes: return f"long sweep needs --long: skipping {spec.name} {spec.axis}={value}" return None