Update loader.py

Author: amillolive

statline/core/adapters/loader.py

@@ -1,6 +1,9 @@
 # statline/core/adapters/loader.py
 from __future__ import annotations
 
+import math
+import os
+import warnings
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, cast
 
@@ -10,6 +13,29 @@
 
 _BASE = Path(__file__).parent / "defs"
 
+# Configurable strictness:
+#   STATLINE_LOADER_STRICT = "1" -> raise on unknown keys / unknown buckets
+#   STATLINE_LOADER_STRICT = "0" (default) -> warn-and-continue with 0.0 / None
+_STRICT = os.environ.get("STATLINE_LOADER_STRICT", "0") not in ("0", "", "false", "False")
+
+
+def _warn(msg: str) -> None:
+    warnings.warn(f"[statline.loader] {msg}", RuntimeWarning, stacklevel=2)
+
+
+def _finite_float(x: Any, default: float = 0.0) -> float:
+    """Coerce to finite float; warn and return default on failure/NaN/inf."""
+    try:
+        v = float(x)
+    except Exception:
+        _warn(f"Non-numeric value '{x}' coerced to {default}")
+        return default
+    if not math.isfinite(v):
+        _warn(f"Non-finite value '{x}' coerced to {default}")
+        return default
+    return v
+
+
 # Allowed top-level keys in an adapter YAML (helps catch typos).
 _ALLOWED_TOP_KEYS: set[str] = {
     "key",
@@ -29,7 +55,9 @@ def _read_yaml_for(name: str) -> Dict[str, Any]:
     if not p.exists():
         p = _BASE / f"{name}.yml"
     if not p.exists():
-        raise FileNotFoundError(f"Adapter spec not found: {name} (expected {name}.yaml or {name}.yml)")
+        raise FileNotFoundError(
+            f"Adapter spec not found: {name} (expected {name}.yaml or {name}.yml)"
+        )
 
     try:
         loaded: Any = yaml.safe_load(p.read_text(encoding="utf-8"))
@@ -40,17 +68,28 @@ def _read_yaml_for(name: str) -> Dict[str, Any]:
     if loaded is None:
         data = {}
     elif isinstance(loaded, dict):
-        # Force Dict[str, Any] shape; cast so keys/values aren’t Unknown to Pylance.
+        # Force Dict[str, Any] shape; cast so keys/values aren’t Unknown to type checkers.
         loaded_map: Mapping[Any, Any] = cast(Mapping[Any, Any], loaded)
         data = {str(k): v for k, v in loaded_map.items()}
     else:
-        raise TypeError(f"Top-level YAML for '{p.name}' must be a mapping (dict), got {type(loaded).__name__}")
+        raise TypeError(
+            f"Top-level YAML for '{p.name}' must be a mapping (dict), got {type(loaded).__name__}"
+        )
 
-    # Unknown top-level keys -> explicit error to avoid silent typos.
+    # Unknown top-level keys -> warn or raise (configurable) to avoid silent typos.
     keys: set[str] = set(data.keys())
     unknown: set[str] = keys.difference(_ALLOWED_TOP_KEYS)
     if unknown:
-        raise KeyError(f"Unknown top-level key(s) in adapter '{name}': {', '.join(sorted(unknown))}")
+        msg = (
+            f"Unknown top-level key(s) in adapter '{name}' ({p}): "
+            f"{', '.join(sorted(unknown))}"
+        )
+        if _STRICT:
+            raise KeyError(msg)
+        _warn(msg + " — ignoring.")
+        for k in list(unknown):
+            data.pop(k, None)
+
     return data
 
 
@@ -62,38 +101,48 @@ def _uniform_weights(buckets: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str,
 
 
 def _as_clamp(v: Any) -> Optional[Tuple[float, float]]:
-    """Normalize clamp configs to (lo, hi) or None. Swaps if lo > hi."""
+    """Normalize clamp configs to (lo, hi) or None. Swaps if lo > hi. Warns on bad forms."""
     if v is None or v is False:
         return None
 
+    def _pair(lo: Any, hi: Any) -> Optional[Tuple[float, float]]:
+        try:
+            a = float(lo)
+            b = float(hi)
+        except Exception:
+            _warn(f"Clamp values '{lo}','{hi}' non-numeric — ignoring clamp")
+            return None
+        if not (math.isfinite(a) and math.isfinite(b)):
+            _warn(f"Clamp values '{lo}','{hi}' non-finite — ignoring clamp")
+            return None
+        if a > b:
+            a, b = b, a
+        if a == b:
+            _warn(f"Clamp with lo==hi ({a}) — ignoring clamp")
+            return None
+        return (a, b)
+
+    # Dict form: {"lo": X, "hi": Y}
+    if isinstance(v, dict) and ("lo" in v and "hi" in v):
+        return _pair(v["lo"], v["hi"])
+
     # Sequence form: [lo, hi] or (lo, hi)
     if isinstance(v, (list, tuple)):
         seq: Sequence[Any] = cast(Sequence[Any], v)
         if len(seq) >= 2:
-            try:
-                lo = float(seq[0])
-                hi = float(seq[1])
-            except (TypeError, ValueError):
-                return None
-            if lo > hi:
-                lo, hi = hi, lo
-            return (lo, hi)
+            return _pair(seq[0], seq[1])
+        _warn(f"Clamp sequence too short: {v} — ignoring clamp")
         return None
 
     # String forms like "0,1" / "0..1" / "0 1"
     if isinstance(v, str):
         parts: List[str] = v.replace(",", " ").replace("..", " ").split()
         if len(parts) >= 2:
-            try:
-                lo = float(parts[0])
-                hi = float(parts[1])
-            except ValueError:
-                return None
-            if lo > hi:
-                lo, hi = hi, lo
-            return (lo, hi)
+            return _pair(parts[0], parts[1])
+        _warn(f"Clamp string malformed: '{v}' — ignoring clamp")
         return None
 
+    _warn(f"Unsupported clamp type {type(v).__name__} — ignoring clamp")
     return None
 
 
@@ -138,6 +187,7 @@ def load_spec(name: str) -> AdapterSpec:
     }
     if not buckets:
         raise ValueError(f"Adapter '{name}': 'buckets' cannot be empty")
+    bucket_names = set(buckets.keys())
 
     # Weights (optional; default to uniform across buckets for 'pri')
     weights_raw: Any = data.get("weights")
@@ -152,11 +202,24 @@ def load_spec(name: str) -> AdapterSpec:
         for profile_any, bw_any in weights_map.items():
             profile = str(profile_any)
             if not isinstance(bw_any, dict):
-                raise TypeError(f"Adapter '{name}': weights profile '{profile}' must be a mapping")
-            inner: Dict[str, float] = {}
+                raise TypeError(
+                    f"Adapter '{name}': weights profile '{profile}' must be a mapping"
+                )
+            # Initialize all known buckets to 0.0 to make omissions explicit.
+            inner: Dict[str, float] = {bk: 0.0 for bk in bucket_names}
             bw_map: Mapping[Any, Any] = cast(Mapping[Any, Any], bw_any)
             for b_any, v_any in bw_map.items():
-                inner[str(b_any)] = float(v_any)
+                bk = str(b_any)
+                if bk not in bucket_names:
+                    msg = (
+                        f"Adapter '{name}': weights profile '{profile}' references "
+                        f"unknown bucket '{bk}'"
+                    )
+                    if _STRICT:
+                        raise KeyError(msg)
+                    _warn(msg + " — treating as 0.0 and ignoring.")
+                    continue
+                inner[bk] = _finite_float(v_any, default=0.0)
             weights_out[profile] = inner
 
     # Penalties (optional; adapter-defined semantics). Keep as {profile: {key: float}}.
@@ -168,11 +231,23 @@ def load_spec(name: str) -> AdapterSpec:
     for profile_any, pw_any in penalties_map.items():
         profile = str(profile_any)
         if not isinstance(pw_any, dict):
-            raise TypeError(f"Adapter '{name}': penalties profile '{profile}' must be a mapping")
+            raise TypeError(
+                f"Adapter '{name}': penalties profile '{profile}' must be a mapping"
+            )
         inner_p: Dict[str, float] = {}
         pw_map: Mapping[Any, Any] = cast(Mapping[Any, Any], pw_any)
         for k_any, v_any in pw_map.items():
-            inner_p[str(k_any)] = float(v_any)
+            bk = str(k_any)
+            if bk not in bucket_names:
+                msg = (
+                    f"Adapter '{name}': penalties profile '{profile}' references "
+                    f"unknown bucket '{bk}'"
+                )
+                if _STRICT:
+                    raise KeyError(msg)
+                _warn(msg + " — dropping penalty.")
+                continue
+            inner_p[bk] = _finite_float(v_any, default=0.0)
         penalties[profile] = inner_p
 
     # Metrics
@@ -189,16 +264,25 @@ def load_spec(name: str) -> AdapterSpec:
             raise KeyError(f"Adapter '{name}': every metric must have a 'key'")
         mkey = str(m["key"])
         if mkey in seen_keys:
-            raise ValueError(f"Adapter '{name}': duplicate metric key '{mkey}'")
+            _warn(
+                f"Adapter '{name}': duplicate metric key '{mkey}' — keeping first, skipping duplicate."
+            )
+            continue
         seen_keys.add(mkey)
 
         bucket_val: Any = m.get("bucket")
         bucket_name: Optional[str] = None
         if bucket_val is not None:
             bname = str(bucket_val)
-            if bname not in buckets:
-                raise KeyError(f"Adapter '{name}': metric '{mkey}' references unknown bucket '{bname}'")
-            bucket_name = bname
+            if bname not in bucket_names:
+                msg = (
+                    f"Adapter '{name}': metric '{mkey}' references unknown bucket '{bname}'"
+                )
+                if _STRICT:
+                    raise KeyError(msg)
+                _warn(msg + " — treating as unscored telemetry (no bucket).")
+            else:
+                bucket_name = bname
 
         metrics.append(
             MetricSpec(
@@ -225,15 +309,21 @@ def load_spec(name: str) -> AdapterSpec:
                 raise KeyError(f"Adapter '{name}': efficiency item missing '{req}'")
         ekey = str(e["key"])
         ebucket = str(e["bucket"])
-        if ebucket not in buckets:
-            raise KeyError(f"Adapter '{name}': efficiency '{ekey}' references unknown bucket '{ebucket}'")
+        if ebucket not in bucket_names:
+            msg = (
+                f"Adapter '{name}': efficiency '{ekey}' references unknown bucket '{ebucket}'"
+            )
+            if _STRICT:
+                raise KeyError(msg)
+            _warn(msg + " — skipping efficiency item.")
+            continue
         eff_list.append(
             EffSpec(
                 key=ekey,
                 make=str(e["make"]),
                 attempt=str(e["attempt"]),
                 bucket=ebucket,
-                min_den=float(e.get("min_den", 1.0)),
+                min_den=_finite_float(e.get("min_den", 1.0), default=1.0),
                 clamp=_as_clamp(e.get("clamp")),
                 invert=bool(e.get("invert", False)),
                 transform=cast(Optional[Mapping[str, Any]], e.get("transform")),
@@ -254,4 +344,4 @@ def load_spec(name: str) -> AdapterSpec:
     )
 
 
-__all__ = ["load_spec"]
+__all__ = ["load_spec"]