Slim package assets and enable URL sidecar fallback

Author: adpena

.cache/repo_districts_campuses.pkl.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e722187d76b37a11030a2862d0aceea4d4ee1f3ac78e6fd70d2cc3f3357eb1d
-size 522
+oid sha256:334d3b5456b405bbd86ba819396bf1735c9b3299c9a217f708d48937b67ebb02
+size 523

.gitignore

@@ -174,6 +174,8 @@ target/
 *.db
 *.pkl
 *.pickle
+/.cache/*.pkl
+/teadata/.cache/*.pkl
 !/.cache/boundaries_*.sqlite
 !/teadata/.cache/boundaries_*.sqlite
 !/.cache/map_payloads_*.sqlite

MANIFEST.in

@@ -1,4 +1,6 @@
-recursive-include teadata/.cache *.pkl
 recursive-include teadata/.cache *.pkl.gz
-recursive-include teadata/.cache *.sqlite
+recursive-include teadata/.cache boundaries_*.sqlite
+recursive-include teadata/.cache entities_*.sqlite
 recursive-include data/private_schools *.csv
+global-exclude *.pkl
+exclude teadata/.cache/map_payloads_*.sqlite

README.md

@@ -184,12 +184,22 @@ PyPI defaults currently documented at:
 
 Reference: <https://docs.pypi.org/project-management/storage-limits/>
 
-Current `teadata` release artifacts for `0.0.118` are above the per-file limit:
+Before packaging trim, `0.0.118` artifacts were above the per-file limit:
 
-- wheel: `dist/teadata-0.0.118-py3-none-any.whl` about `448 MB`
-- sdist: `dist/teadata-0.0.118.tar.gz` about `446 MB`
+- wheel: about `448 MB`
+- sdist: about `446 MB`
 
-These exceed the default 100 MB file cap because large `.cache` snapshot/store artifacts are packaged into both distributions.
+Current slimmed artifacts are below the limit:
+
+- wheel: `dist/teadata-0.0.118-py3-none-any.whl` about `74 MB`
+- sdist: `dist/teadata-0.0.118.tar.gz` about `72 MB`
+
+To stay under PyPI file limits while preserving runtime behavior:
+
+- PyPI package data now includes compressed snapshots (`.pkl.gz`) and selected sidecars (`boundaries_*.sqlite`, `entities_*.sqlite`).
+- Uncompressed `.pkl` files are excluded from distributions.
+- `map_payloads_*.sqlite` is excluded from distributions; provide it at runtime via `TEADATA_MAP_STORE` or `TEADATA_MAP_STORE_URL`.
+- URL-based store discovery supports snapshot-derived sidecar paths, so `TEADATA_*_URL` can hydrate missing local sidecars automatically.
 
 ## Release Policy
 

pyproject.toml

@@ -98,10 +98,10 @@ include = ["teadata*"]
 exclude = ["tests*", "docs*", "examples*"]
 
 [tool.setuptools.package-data]
-teadata = [".cache/*.pkl", ".cache/*.pkl.gz", ".cache/*.sqlite"]
+teadata = [".cache/*.pkl.gz", ".cache/boundaries_*.sqlite", ".cache/entities_*.sqlite"]
 
 [tool.setuptools]
-include-package-data = true
+include-package-data = false
 
 [project.urls]
 "Homepage" = "https://github.com/adpena/teadata"

teadata/.cache/repo_districts_campuses.pkl.gz

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e722187d76b37a11030a2862d0aceea4d4ee1f3ac78e6fd70d2cc3f3357eb1d
-size 522
+oid sha256:334d3b5456b405bbd86ba819396bf1735c9b3299c9a217f708d48937b67ebb02
+size 523

teadata/boundary_store.py

@@ -77,18 +77,24 @@ def _resolve_boundary_store(path: Path, source: str) -> Optional[Path]:
     return resolved
 
 
+def _should_attempt_boundary_resolution(path: Path) -> bool:
+    if path.exists() and path.is_file():
+        return True
+    return bool(os.environ.get("TEADATA_BOUNDARY_STORE_URL"))
+
+
 def discover_boundary_store(explicit: str | Path | None = None) -> Optional[Path]:
     if explicit:
         p = Path(explicit)
-        if p.exists() and p.is_file():
+        if _should_attempt_boundary_resolution(p):
             resolved = _resolve_boundary_store(p, "explicit")
             if resolved:
                 return resolved
 
     env = os.environ.get("TEADATA_BOUNDARY_STORE")
     if env:
         p = Path(env)
-        if p.exists() and p.is_file():
+        if _should_attempt_boundary_resolution(p):
             resolved = _resolve_boundary_store(p, "env")
             if resolved:
                 return resolved
@@ -98,7 +104,7 @@ def discover_boundary_store(explicit: str | Path | None = None) -> Optional[Path
             snap = _discover_snapshot()
             if snap:
                 candidate = boundary_store_path_for_snapshot(Path(snap))
-                if candidate and candidate.exists():
+                if candidate and _should_attempt_boundary_resolution(candidate):
                     resolved = _resolve_boundary_store(candidate, "snapshot")
                     if resolved:
                         return resolved

teadata/entity_store.py

@@ -72,18 +72,24 @@ def _resolve_entity_store(path: Path, source: str) -> Optional[Path]:
     return resolved
 
 
+def _should_attempt_entity_resolution(path: Path) -> bool:
+    if path.exists() and path.is_file():
+        return True
+    return bool(os.environ.get("TEADATA_ENTITY_STORE_URL"))
+
+
 def discover_entity_store(explicit: str | Path | None = None) -> Optional[Path]:
     if explicit:
         p = Path(explicit)
-        if p.exists() and p.is_file():
+        if _should_attempt_entity_resolution(p):
             resolved = _resolve_entity_store(p, "explicit")
             if resolved:
                 return resolved
 
     env = os.environ.get("TEADATA_ENTITY_STORE")
     if env:
         p = Path(env)
-        if p.exists() and p.is_file():
+        if _should_attempt_entity_resolution(p):
             resolved = _resolve_entity_store(p, "env")
             if resolved:
                 return resolved
@@ -93,7 +99,7 @@ def discover_entity_store(explicit: str | Path | None = None) -> Optional[Path]:
             snap = _discover_snapshot()
             if snap:
                 candidate = entity_store_path_for_snapshot(Path(snap))
-                if candidate and candidate.exists():
+                if candidate and _should_attempt_entity_resolution(candidate):
                     resolved = _resolve_entity_store(candidate, "snapshot")
                     if resolved:
                         return resolved

teadata/map_store.py

@@ -65,18 +65,24 @@ def _resolve_map_store(path: Path, source: str) -> Optional[Path]:
     return resolved
 
 
+def _should_attempt_map_resolution(path: Path) -> bool:
+    if path.exists() and path.is_file():
+        return True
+    return bool(os.environ.get("TEADATA_MAP_STORE_URL"))
+
+
 def discover_map_store(explicit: str | Path | None = None) -> Optional[Path]:
     if explicit:
         p = Path(explicit)
-        if p.exists() and p.is_file():
+        if _should_attempt_map_resolution(p):
             resolved = _resolve_map_store(p, "explicit")
             if resolved:
                 return resolved
 
     env = os.environ.get("TEADATA_MAP_STORE")
     if env:
         p = Path(env)
-        if p.exists() and p.is_file():
+        if _should_attempt_map_resolution(p):
             resolved = _resolve_map_store(p, "env")
             if resolved:
                 return resolved
@@ -86,7 +92,7 @@ def discover_map_store(explicit: str | Path | None = None) -> Optional[Path]:
             snap = _discover_snapshot()
             if snap:
                 candidate = map_store_path_for_snapshot(Path(snap))
-                if candidate and candidate.exists():
+                if candidate and _should_attempt_map_resolution(candidate):
                     resolved = _resolve_map_store(candidate, "snapshot")
                     if resolved:
                         return resolved

tests/test_store_discovery.py

@@ -50,3 +50,72 @@ def fake_resolve(path: Path, source: str):
 
     assert found == candidate
     assert seen_sources == ["cwd-cache"]
+
+
+@pytest.mark.parametrize(
+    (
+        "module",
+        "discover_name",
+        "resolve_name",
+        "env_name",
+        "url_env_name",
+        "snapshot_path_name",
+    ),
+    [
+        (
+            map_store,
+            "discover_map_store",
+            "_resolve_map_store",
+            "TEADATA_MAP_STORE",
+            "TEADATA_MAP_STORE_URL",
+            "map_store_path_for_snapshot",
+        ),
+        (
+            boundary_store,
+            "discover_boundary_store",
+            "_resolve_boundary_store",
+            "TEADATA_BOUNDARY_STORE",
+            "TEADATA_BOUNDARY_STORE_URL",
+            "boundary_store_path_for_snapshot",
+        ),
+        (
+            entity_store,
+            "discover_entity_store",
+            "_resolve_entity_store",
+            "TEADATA_ENTITY_STORE",
+            "TEADATA_ENTITY_STORE_URL",
+            "entity_store_path_for_snapshot",
+        ),
+    ],
+)
+def test_discover_store_uses_snapshot_candidate_when_url_is_configured(
+    monkeypatch,
+    tmp_path: Path,
+    module,
+    discover_name: str,
+    resolve_name: str,
+    env_name: str,
+    url_env_name: str,
+    snapshot_path_name: str,
+):
+    monkeypatch.delenv(env_name, raising=False)
+    monkeypatch.setenv(url_env_name, "https://example.com/store.sqlite")
+
+    snapshot = tmp_path / "repo_test.pkl.gz"
+    expected = getattr(module, snapshot_path_name)(snapshot)
+    assert expected is not None
+
+    seen_sources: list[str] = []
+
+    def fake_resolve(path: Path, source: str):
+        seen_sources.append(source)
+        return path
+
+    monkeypatch.setattr(module, "_discover_snapshot", lambda: snapshot)
+    monkeypatch.setattr(module, resolve_name, fake_resolve)
+
+    discover = getattr(module, discover_name)
+    found = discover()
+
+    assert found == expected
+    assert seen_sources == ["snapshot"]