Merge pull request #149 from DataFog/fix/4.5-release-review

fix: 4.5.0 release-review fixes — detection parity, changelog, release metadata

Author: sidmohan0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.4.0a5
+current_version = 4.5.0b5
 commit = True
 tag = True
 tag_name = v{new_version}

.pre-commit-config.yaml

@@ -36,6 +36,7 @@ repos:
         exclude: |
           (?x)^(
             .venv|
+            docs/audit/.*|
             .*\.github/workflows/.*\.ya?ml$
           )$
 

CHANGELOG.MD

@@ -1,9 +1,27 @@
 # ChangeLog
 
-## [Unreleased]
+## [2026-07-02]
 
 ### `datafog-python` [4.5.0]
 
+#### Behavior Changes Since 4.4.0
+
+- Agent guardrail helpers (`sanitize`, `scan_prompt`, `filter_output`,
+  `create_guardrail`, and the `Guardrail` class) now default to
+  `engine="regex"` instead of `engine="smart"`, matching the top-level
+  `scan`/`redact`/`protect` defaults. This keeps the core install from
+  probing optional NLP dependencies, but means NER-backed entities
+  (PERSON, ORGANIZATION, LOCATION) are no longer detected by these
+  helpers unless requested. **Migration:** pass `engine="smart"` to
+  restore the 4.4.0 behavior (requires `datafog[nlp]` or
+  `datafog[nlp-advanced]`).
+- `DataFog.detect()` / `DataFog.scan_text()` result dictionaries now
+  contain keys only for the labels active under the configured locales:
+  the seven base labels by default, plus the `DE_*` labels when
+  constructed with `locales=["de"]`.
+- Supported Python versions are `>=3.10,<3.14`. Python 3.14 is not yet
+  certified and is intentionally excluded from this release.
+
 #### Release Thesis
 
 - Frames 4.5.0 as a focused, lightweight text PII screening release rather
@@ -25,11 +43,15 @@
 
 - Adds regex-only German structured PII support without adding core
   dependencies.
-- Detects German VAT IDs and German IBANs by default because their country-code
-  structure is precise enough for default screening.
-- Enables broader German identifiers only through `locales=["de"]` or explicit
-  entity selection, including German tax IDs, pension insurance numbers,
-  postal codes, passport numbers, and residence permit numbers.
+- All German identifiers are locale-gated: they activate only through
+  `locales=["de"]` or explicit entity selection. This covers German VAT
+  IDs, IBANs, tax IDs, pension insurance numbers, postal codes, passport
+  numbers, and residence permit numbers. Default (no-locale) detection
+  behavior is unchanged from 4.4.0.
+- When German locale support is active, overlapping matches (e.g. the
+  nine-digit run inside a VAT ID also matching the generic SSN pattern)
+  are resolved by engine-level span-overlap suppression in favor of the
+  more specific German label.
 
 #### Optional Profiles And Python 3.13
 
@@ -61,9 +83,9 @@
 - Adds a 4.5 release-readiness checklist covering docs build, formatting,
   core no-network checks, install-profile smoke checks, German regex tests,
   broad non-slow tests, package build checks, and final CI status.
-- Clarifies the version alignment path: the development package remains
-  `4.4.0a5` until stable release promotion, and the final stable release should
-  publish as `4.5.0`.
+- Clarifies the version alignment path: development prereleases publish as
+  `4.5.0aN`/`4.5.0bN` from `dev`, and the stable release publishes as
+  `4.5.0` via the Release workflow's `stable` dispatch from `main`.
 
 ## [2026-02-13]
 

datafog/agent.py

@@ -44,7 +44,12 @@ def filter(self, text: str) -> RedactResult:
 
 @dataclass
 class Guardrail:
-    """Reusable text guardrail for wrapping LLM prompts and outputs."""
+    """Reusable text guardrail for wrapping LLM prompts and outputs.
+
+    Defaults to the lightweight regex engine (changed from "smart" in 4.5.0)
+    so the core install never probes optional NLP dependencies; pass
+    ``engine="smart"`` to restore NER-backed detection.
+    """
 
     entity_types: Optional[list[str]] = None
     locales: Optional[list[str]] = None
@@ -119,6 +124,10 @@ def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str:
     One-liner PII removal.
 
     Returns the redacted text only.
+
+    Uses the lightweight regex engine by default (changed from "smart" in
+    4.5.0); pass ``engine="smart"`` for NER-backed detection, which requires
+    the optional NLP extras.
     """
     result = scan_and_redact(text=text, engine=engine, **kwargs)
     return result.redacted_text
@@ -127,13 +136,19 @@ def sanitize(text: str, engine: str = "regex", **kwargs: Any) -> str:
 def scan_prompt(prompt: str, engine: str = "regex", **kwargs: Any) -> ScanResult:
     """
     Scan an LLM prompt for PII without modifying the input text.
+
+    Uses the lightweight regex engine by default (changed from "smart" in
+    4.5.0); pass ``engine="smart"`` for NER-backed detection.
     """
     return scan(prompt, engine=engine, **kwargs)
 
 
 def filter_output(output: str, engine: str = "regex", **kwargs: Any) -> RedactResult:
     """
     Scan and redact PII from model output before returning to users.
+
+    Uses the lightweight regex engine by default (changed from "smart" in
+    4.5.0); pass ``engine="smart"`` for NER-backed detection.
     """
     return scan_and_redact(output, engine=engine, **kwargs)
 

datafog/main.py

@@ -184,7 +184,10 @@ def detect(self, text: str) -> dict:
         _start = _time.monotonic()
 
         scan_result = scan(text=text, engine="regex", locales=self.locales)
-        result = {label: [] for label in RegexAnnotator.LABELS}
+        # Only pre-populate keys for labels active under the configured
+        # locales so the default output shape matches v4.4.0 (no DE_* keys
+        # unless German locale support is enabled).
+        result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)}
         legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"}
         for entity in scan_result.entities:
             label = legacy_map.get(entity.type, entity.type)

datafog/processing/text_processing/regex_annotator/regex_annotator.py

@@ -53,7 +53,7 @@ def __init__(
         enabled_labels: Iterable[str] | None = None,
     ):
         self.locales = self._normalize_locales(locales)
-        self.active_labels = self._resolve_active_labels(enabled_labels)
+        self.active_labels = self.active_labels_for(self.locales, enabled_labels)
 
         # Compile all patterns once at initialization
         all_patterns: Dict[str, Pattern] = {
@@ -100,20 +100,19 @@ def __init__(
             ),
             # SSN pattern - U.S. Social Security Number
             # Supports dashed and no-dash formats.
+            # Note: overlaps with locale-gated labels (e.g. the nine-digit run
+            # inside a DE_VAT_ID) are resolved by the engine's span-overlap
+            # suppression, not here, so default (EN) detection keeps v4.4.0
+            # behavior even when German labels are active.
             "SSN": re.compile(
                 r"""
+                (?<!\d)
                 (?:
-                    (?<!\d)
                     (?!000|666)\d{3}-(?!00)\d{2}-(?!0000)\d{4}
-                    (?!\d)
                     |
-                    (?<![A-Za-z0-9])
-                    (?<!DE)
-                    (?<!DE\s)
-                    (?<!DE-)
                     (?!000|666)\d{3}(?!00)\d{2}(?!0000)\d{4}
-                    (?![A-Za-z0-9])
                 )
+                (?!\d)
                 """,
                 re.IGNORECASE | re.MULTILINE | re.VERBOSE,
             ),
@@ -347,13 +346,19 @@ def _normalize_locales(cls, locales: str | Iterable[str] | None) -> tuple[str, .
             normalized.append(value)
         return tuple(dict.fromkeys(normalized))
 
-    def _resolve_active_labels(self, enabled_labels: Iterable[str] | None) -> list[str]:
-        active = set(self.DEFAULT_LABELS)
-        for locale in self.locales:
-            active.update(self.LOCALE_LABELS[locale])
+    @classmethod
+    def active_labels_for(
+        cls,
+        locales: str | Iterable[str] | None = None,
+        enabled_labels: Iterable[str] | None = None,
+    ) -> list[str]:
+        """Resolve the labels active for the given locales and explicit labels."""
+        active = set(cls.DEFAULT_LABELS)
+        for locale in cls._normalize_locales(locales):
+            active.update(cls.LOCALE_LABELS[locale])
         if enabled_labels is not None:
             active.update(label.strip().upper() for label in enabled_labels)
-        return [label for label in self.LABELS if label in active]
+        return [label for label in cls.LABELS if label in active]
 
     @staticmethod
     def _match_text(match: Match[str]) -> str:

docs/v45-release-readiness.rst

@@ -47,10 +47,13 @@ Use this framing for the GitHub release notes and package announcement:
 
 Call out these user-facing points:
 
-* German VAT IDs and German IBANs are detected by default in the regex engine.
-* Broader German identifiers such as tax IDs, postal codes, passport numbers,
-  residence permit numbers, and pension insurance numbers require
-  ``locales=["de"]`` or explicit entity selection.
+* German structured identifiers — VAT IDs, IBANs, tax IDs, postal codes,
+  passport numbers, residence permit numbers, and pension insurance numbers —
+  are locale-gated and require ``locales=["de"]`` or explicit entity
+  selection. Default (no-locale) detection behavior is unchanged from 4.4.0.
+* Guardrail helpers (``sanitize``, ``scan_prompt``, ``filter_output``,
+  ``create_guardrail``) now default to the regex engine; pass
+  ``engine="smart"`` to restore 4.4.0's NER-backed helper behavior.
 * OCR and Spark remain supported optional surfaces. They are not deprecated,
   but their broader overhaul is deferred beyond 4.5.
 * Telemetry remains disabled unless ``DATAFOG_TELEMETRY=1`` is set.

tests/test_agent_api.py

@@ -2,10 +2,22 @@
 
 from __future__ import annotations
 
+import inspect
+
 import pytest
 
 import datafog
-from datafog.agent import GuardrailBlockedError
+from datafog.agent import Guardrail, GuardrailBlockedError
+
+
+def test_agent_helpers_default_to_regex_engine() -> None:
+    """4.5 lean-core contract: guardrail helpers stay on the regex path
+    unless an engine is explicitly requested (changed from "smart" in 4.5.0;
+    see CHANGELOG behavior changes)."""
+    assert Guardrail().engine == "regex"
+    assert datafog.create_guardrail().engine == "regex"
+    for helper in (datafog.sanitize, datafog.scan_prompt, datafog.filter_output):
+        assert inspect.signature(helper).parameters["engine"].default == "regex"
 
 
 def test_sanitize_redacts_structured_pii() -> None:

tests/test_de_pii_regex.py

@@ -134,16 +134,22 @@ def test_redaction_and_service_locale_support() -> None:
 def test_german_vat_redaction_suppresses_inner_generic_ssn_match(
     text: str, vat_text: str
 ) -> None:
+    # Default (no locale): v4.4.0 parity — the bare nine-digit run still
+    # matches the generic SSN pattern even when prefixed by a country code.
     scan_result = scan(text, engine="regex")
-    assert scan_result.entities == []
+    assert [(entity.type, entity.text) for entity in scan_result.entities] == [
+        ("SSN", "123456789")
+    ]
 
+    # German locale: the longer DE_VAT_ID span wins via the engine's
+    # span-overlap suppression, so the inner SSN match is dropped.
     locale_scan_result = scan(text, engine="regex", locales=["de"])
     assert [(entity.type, entity.text) for entity in locale_scan_result.entities] == [
         ("DE_VAT_ID", vat_text)
     ]
 
     default_redaction = scan_and_redact(text, engine="regex")
-    assert default_redaction.redacted_text == text
+    assert default_redaction.redacted_text == text.replace("123456789", "[SSN_1]")
 
     redaction = scan_and_redact(text, engine="regex", locales=["de"])
     assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]")

tests/test_regex_annotator.py

@@ -371,3 +371,18 @@ def test_annotation_result_format():
 
     assert len(ssn_spans) >= 1
     assert ssn_spans[0].text == "123-45-6789"
+
+
+def test_ssn_detection_keeps_v44_behavior_for_country_prefixed_digits():
+    """Regression guard: bare nine-digit runs after a country prefix must
+    still match SSN when no locale is configured (v4.4.0 parity). The
+    DE_VAT_ID overlap is resolved by engine-level span suppression only
+    when German locale support is active, never by weakening the base
+    SSN pattern."""
+    annotator = RegexAnnotator()
+    for text in (
+        "Reference DE 123456789 was issued.",
+        "Reference DE-123456789 was issued.",
+        "Reference DE123456789 was issued.",
+    ):
+        assert annotator.annotate(text)["SSN"] == ["123456789"], text