From fa99a4bcd66c25a0a254429dfbba5c652d4db7b4 Mon Sep 17 00:00:00 2001 From: Dat Nguyen Date: Sat, 13 Jun 2026 15:32:58 +0700 Subject: [PATCH 1/3] feat: add Project Health Check (six dbt-project-evaluator dimensions) --- .claude/commands/dbdocs-code-review.md | 18 + .claude/commands/dbdocs-pr.md | 23 + .claude/design_patterns.md | 148 +++++ .claude/settings.json | 3 + .claude/skills/dbdocs-code-review/SKILL.md | 310 +++++++++++ .claude/skills/dbdocs-pr/SKILL.md | 143 +++++ .github/ISSUE_TEMPLATE/bug_report.yml | 103 ++++ .github/ISSUE_TEMPLATE/config.yml | 8 + .github/ISSUE_TEMPLATE/feature_request.yml | 66 +++ .github/PULL_REQUEST_TEMPLATE.md | 48 ++ CLAUDE.md | 4 +- README.md | 31 +- dbdocs.yml.example | 48 ++ dbdocs/cli/main.py | 20 +- dbdocs/core/config.py | 14 + dbdocs/extract/health/__init__.py | 12 + dbdocs/extract/health/dimensions.py | 262 +++++++++ dbdocs/extract/health/extractor.py | 299 ++++++++++ dbdocs/extract/health/rules/__init__.py | 106 ++++ dbdocs/extract/health/rules/base.py | 43 ++ .../health/rules/dimensions/__init__.py | 7 + .../health/rules/dimensions/documentation.py | 71 +++ .../health/rules/dimensions/governance.py | 75 +++ .../health/rules/dimensions/modeling.py | 241 +++++++++ .../health/rules/dimensions/performance.py | 65 +++ .../health/rules/dimensions/structure.py | 79 +++ .../health/rules/dimensions/testing.py | 51 ++ dbdocs/extract/health/rules/registry.py | 155 ++++++ dbdocs/site/builder.py | 26 +- dbdocs/site/bundle/assets/css/icons.css | 35 ++ dbdocs/site/bundle/assets/css/style.css | 151 +++++- dbdocs/site/bundle/assets/graph/index.js | 24 +- dbdocs/site/bundle/assets/js/data.js | 10 +- dbdocs/site/bundle/assets/js/service.js | 128 ++++- dbdocs/site/bundle/assets/js/ui.js | 409 ++++++++++++-- dbdocs/site/bundle/index.html | 7 + docs/dbdocs-demo.yml | 58 ++ docs/index.md | 52 +- docs/plans/scale-mobile-e2e-branding.md | 284 ---------- frontend/src/components/GraphApp.tsx | 71 ++- frontend/src/main.tsx | 11 +- frontend/test/e2e/spa.spec.ts | 309 ++++++++++- frontend/test/unit/buildDagHash.test.ts | 34 ++ pyproject.toml | 4 +- tests/conftest.py | 7 + tests/fixtures/jaffle_shop/run_results.json | 442 +++++++++++++++ .../integration/test_generate_jaffle_shop.py | 19 + tests/unit/cli/test_cli.py | 47 +- tests/unit/core/test_config.py | 12 + tests/unit/extract/health/__init__.py | 0 tests/unit/extract/health/conftest.py | 17 + tests/unit/extract/health/test_dimensions.py | 293 ++++++++++ tests/unit/extract/health/test_extractor.py | 497 +++++++++++++++++ tests/unit/extract/health/test_plugins.py | 199 +++++++ tests/unit/extract/health/test_rules.py | 509 ++++++++++++++++++ tests/unit/site/test_builder.py | 112 +++- uv.lock | 214 ++++---- 57 files changed, 5960 insertions(+), 474 deletions(-) create mode 100644 .claude/commands/dbdocs-code-review.md create mode 100644 .claude/commands/dbdocs-pr.md create mode 100644 .claude/skills/dbdocs-code-review/SKILL.md create mode 100644 .claude/skills/dbdocs-pr/SKILL.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 dbdocs/extract/health/__init__.py create mode 100644 dbdocs/extract/health/dimensions.py create mode 100644 dbdocs/extract/health/extractor.py create mode 100644 dbdocs/extract/health/rules/__init__.py create mode 100644 dbdocs/extract/health/rules/base.py create mode 100644 dbdocs/extract/health/rules/dimensions/__init__.py create mode 100644 dbdocs/extract/health/rules/dimensions/documentation.py create mode 100644 dbdocs/extract/health/rules/dimensions/governance.py create mode 100644 dbdocs/extract/health/rules/dimensions/modeling.py create mode 100644 dbdocs/extract/health/rules/dimensions/performance.py create mode 100644 dbdocs/extract/health/rules/dimensions/structure.py create mode 100644 dbdocs/extract/health/rules/dimensions/testing.py create mode 100644 dbdocs/extract/health/rules/registry.py create mode 100644 dbdocs/site/bundle/assets/css/icons.css delete mode 100644 docs/plans/scale-mobile-e2e-branding.md create mode 100644 frontend/test/unit/buildDagHash.test.ts create mode 100644 tests/fixtures/jaffle_shop/run_results.json create mode 100644 tests/unit/extract/health/__init__.py create mode 100644 tests/unit/extract/health/conftest.py create mode 100644 tests/unit/extract/health/test_dimensions.py create mode 100644 tests/unit/extract/health/test_extractor.py create mode 100644 tests/unit/extract/health/test_plugins.py create mode 100644 tests/unit/extract/health/test_rules.py diff --git a/.claude/commands/dbdocs-code-review.md b/.claude/commands/dbdocs-code-review.md new file mode 100644 index 0000000..8d98969 --- /dev/null +++ b/.claude/commands/dbdocs-code-review.md @@ -0,0 +1,18 @@ +--- +description: Review the current dbdocs change for consistency, Python pluggability, design-pattern alignment, the 3-tier bundle-JS contract, complexity/scale, and the generated-SPA UX & accessibility. +--- + +Use the **dbdocs-code-review** skill to review dbdocs changes against this +codebase's documented patterns. + +Target (in order of precedence): +1. If `$ARGUMENTS` names a PR number, branch, or path, review that. +2. Otherwise review the working-tree diff (`git diff` + untracked new files). + +Follow the skill's six dimensions — **Consistency**, **Pluggable (Python +modules)**, **Align with design patterns**, **3-tier bundle JS**, **Complexity / +scale**, **UX & accessibility** — run the gates (report, don't fix), and emit the +severity-graded findings report in the skill's output format. + +Do not modify any files during the review. End by offering to apply the +behavior-neutral findings (all / high-severity subset / none). diff --git a/.claude/commands/dbdocs-pr.md b/.claude/commands/dbdocs-pr.md new file mode 100644 index 0000000..0f81825 --- /dev/null +++ b/.claude/commands/dbdocs-pr.md @@ -0,0 +1,23 @@ +--- +description: Open a GitHub PR for the current branch with a body that follows .github/PULL_REQUEST_TEMPLATE.md, filled from the actual diff and with the checklist verified against the repo gates. +--- + +Use the **dbdocs-pr** skill to open a pull request for the current branch. + +Target (in order of precedence): +1. If `$ARGUMENTS` names a base branch, an issue to close, draft, or a title, use it. +2. Otherwise PR the current branch against the repo's default branch. + +Follow the skill: run the **pre-flight** and, when HEAD is `main` (or the repo +default), **create a feature branch** carrying the working tree onto a +`/` name derived from the change (confirm the name first; use +`$ARGUMENTS` verbatim if a name is given). Then run the **gates** (`ruff` + +`pytest` at 100% coverage) and fold the real results into the checklist, +**compose the body from `.github/PULL_REQUEST_TEMPLATE.md`** — tick the one +correct Type box, every touched Area, and only the checklist items you actually +verified — then `git push -u origin HEAD` and `gh pr create`. + +Creating a PR is outward-facing: confirm anything ambiguous (base, draft vs +ready, which commits belong) before pushing. If the gates are red, stop and offer +to fix or to open a **draft** with the failures called out — never tick a red box. +End with the PR URL. diff --git a/.claude/design_patterns.md b/.claude/design_patterns.md index 5d9650b..1426a91 100644 --- a/.claude/design_patterns.md +++ b/.claude/design_patterns.md @@ -9,17 +9,44 @@ authoritative; grep it. - [Design patterns](#design-patterns) - [Table of contents](#table-of-contents) - [Pipeline-stage package layout](#pipeline-stage-package-layout) + - [Theory](#theory) + - [Example](#example) - [One data dict + external gzip payload](#one-data-dict--external-gzip-payload) + - [Theory](#theory-1) + - [Example](#example-1) - [SPA 3-tier ES modules (data → service → ui)](#spa-3-tier-es-modules-data--service--ui) + - [Theory](#theory-2) + - [Example](#example-2) - [Config object from dbdocs.yml](#config-object-from-dbdocsyml) + - [Theory](#theory-3) + - [Example](#example-3) - [Centralized artifact loading + the schema\_ gotcha](#centralized-artifact-loading--the-schema_-gotcha) + - [Theory](#theory-4) + - [Example](#example-4) - [Manifest-base columns, catalog-enriched](#manifest-base-columns-catalog-enriched) + - [Theory](#theory-5) + - [Example](#example-5) - [Fail-soft, parallel column-level lineage](#fail-soft-parallel-column-level-lineage) + - [Theory](#theory-6) + - [Example](#example-6) - [Windowed graph rendering](#windowed-graph-rendering) + - [Theory](#theory-7) + - [Example](#example-7) - [Bundled SPA directory resolution](#bundled-spa-directory-resolution) + - [Theory](#theory-8) + - [Example](#example-8) - [Versioned deploy without mike](#versioned-deploy-without-mike) + - [Theory](#theory-9) + - [Example](#example-9) - [Click group entrypoint](#click-group-entrypoint) + - [Theory](#theory-10) + - [Example](#example-10) - [Singleton colored logger](#singleton-colored-logger) + - [Theory](#theory-11) + - [Example](#example-11) + - [Always-built artifact-derived data-dict section (Health Check)](#always-built-artifact-derived-data-dict-section-health-check) + - [Theory](#theory-12) + - [Example](#example-12) ## Pipeline-stage package layout @@ -450,3 +477,124 @@ if len(logger.handlers) == 0: # pragma: no cover - import-time handler guard ``` - `dbdocs/core/log.py` — `class LogFormatter`, `logger = logging.getLogger("dbdocs")` + +## Always-built artifact-derived data-dict section (Health Check) + +### Theory + +Some data-dict sections are derived from an **extra artifact** (e.g. +`run_results.json`) that isn't always present. The section is **always built** +(no opt-in flag) and **fail-soft to empty** when the artifact is missing; the SPA +decides whether to *surface* it based on whether it holds anything. The pattern: + +1. An optional path field on `DbDocsConfig` (e.g. `run_results: str | None = + None`) added to `_NON_METADATA_FIELDS` (build-control, not display metadata). + No `bool` enable-flag — the section is unconditional. +2. A `--run-results` CLI option (path override) that writes back to the config + object before `ReportBuilder` is called — same as the existing `--dialect` + pattern. No `--evaluator/--no-evaluator` on/off flag. +3. A dedicated extractor class in `dbdocs/extract/` (e.g. `HealthCheckExtractor`) + that reads the artifact and returns a plain dict ready for the data dict. The + artifact is parsed with **`artifact-parser`** (`artifact_parser.dbt.parse_run_results`), + a versioned-Pydantic parser covering run-results schema v1–v6 — *not* dbterd + and *not* the manifest Pydantic parser; a schema/enum rename in a new dbt + release then surfaces as a caught error rather than a silent mis-read. The + parsed `Result.status` is an enum (`.value` is the plain string — never `str()` + it). **Fail-soft**: every IO + parse error is caught with a *specific* type + (`FileNotFoundError`/`OSError`, `json.JSONDecodeError`, and the parser's + `ArtifactParserError`/Pydantic `ValidationError`), logged, and returns an + empty-but-enabled section — a missing file must never sink `generate`. Health + works from an **ordinary `dbt build`/`dbt test`** — no extra dbt package: every + `test.*` result is a finding, bucketed by test type (`not_null`/`unique` → + integrity, `relationships` → referential, `accepted_values` → validity, …). The + type isn't in `run_results.json` — it lives in the **manifest** + (`test_metadata.name`, `column_name`, `attached_node`), so the extractor is + passed the dbterd-parsed manifest and enriches each finding from it, falling + back to inferring the type from the `unique_id` when no manifest/node is found. +4. `ReportBuilder.build_data()` always resolves the path (via `_resolve_within_cwd` + with fail-soft on escape, defaulting to `/`), calls the + extractor, and adds the top-level `health` key unconditionally. +5. The SPA's `data.js normalize()` defaults the key (`health: {enabled: false}`) + so the ui tier never crashes if it's absent in an old payload. `service.js` + exposes pure accessors (DOM-free); **`healthEnabled()` keys off `summary.total + > 0`** (not a config flag) — so an empty section (no `run_results.json`) means + no nav entry / overview card, while a populated one surfaces them. `ui.js` + guards rendering behind `healthEnabled()`. + +Do not invent a second data-hand-off channel. Extend the single data dict. + +### Example + +```python +# dbdocs/site/builder.py — health is always built; fail-soft path resolution +data = { + # ... other sections ... + "health": HealthCheckExtractor( + self._resolve_run_results_path(), manifest + ).extract(), +} + +def _resolve_run_results_path(self) -> str: + if self.config.run_results: + try: + return str(_resolve_within_cwd(self.config.run_results, "run_results")) + except DbDocsConfigError: + logger.warning("run_results path %r escapes the project directory ...", ...) + return str(Path(self.config.target_path) / "run_results.json") +``` + +```python +# dbdocs/extract/health/extractor.py — parse via artifact-parser; specific exception types only +def _load_results(self) -> "list | None": + try: + text = self._path.read_text(encoding="utf-8") + except FileNotFoundError: + logger.warning("Health check: run_results.json not found at %s — skipping.", ...) + return None + except OSError as exc: + logger.warning("Health check: could not read %s: %s — skipping.", ...) + return None + try: + raw = json.loads(text) + except json.JSONDecodeError as exc: + logger.warning("Health check: could not parse %s: %s — skipping.", ...) + return None + try: + run_results = parse_run_results(raw) # artifact_parser.dbt + except (ArtifactParserError, ValidationError) as exc: + logger.warning("Health check: %s is not a valid run_results artifact ...", ...) + return None + return list(run_results.results) +``` + +The health data dict has three parts: `dimensions` (the six DPE dimensions, each +`{issues, checked, findings}`, computed from the manifest by the rule engine), +`testResults` (the per-test pass/fail detail from `run_results.json`, or `null`), +and `note` (set when `run_results.json` was absent). The SPA renders a **scorecard ++ collapsible dimension** page; the per-test detail is shown on each **model +page** (split into Data tests / Unit tests), not on the Health page. + +The rule engine is **a package**: the dimension modules live under +`extract/health/rules/dimensions/{testing,modeling,documentation,structure,performance,governance}.py`, +with a shared `base.py` (public `finding()`/`docs_url()`, `DEFAULT_THRESHOLDS`, +`LAYER_PREFIXES`, `NON_PHYSICAL` — no cross-module private imports), `registry.py` +(the impl: `DIMENSION_RULES` + the plugin API), and a **thin** `__init__.py` +facade (re-export only — no impl). It is **configurable + pluggable** via +`dbdocs.yml` `health:` — `thresholds` (read through `graph.threshold(name)`), +`disable` / `disable_dimensions`, and `rules_module`; plus the `dbdocs.health_rules` +entry point. `register_rule(dimension)` (decorator or call) appends to the +module-global `DIMENSION_RULES` — tests must `reset_rules()` between cases (an autouse conftest +fixture does this). + +- `dbdocs/core/config.py` — `run_results`, `health` fields, `_NON_METADATA_FIELDS` (no `evaluator` flag — health is always built) +- `dbdocs/cli/main.py` — `--run-results` path override (no on/off flag) +- `dbdocs/extract/health/extractor.py` — `class HealthCheckExtractor`, `TEST_CATEGORIES`, `_is_test_result`, `_is_unit_test`, `_resolve_metadata`, `_unit_test_model`, `_status_value`, `parse_run_results` (from `artifact_parser.dbt`) +- `dbdocs/extract/health/dimensions.py` — `class ManifestGraph` (adjacency + `threshold`/`layer`/`materialization`/`access`/`tests_for`), `class DimensionAnalyzer` (config wiring, plugin load, disable lists) +- `dbdocs/extract/health/rules/` — `base.py` (public `finding`, `docs_url`, `DEFAULT_THRESHOLDS`, `LAYER_PREFIXES`, `NON_PHYSICAL`), `dimensions/` (one module per dimension), `registry.py` (`DIMENSION_RULES`, `register_rule`, `reset_rules`, `load_entry_point_rules`, `load_rules_module`, `ENTRY_POINT_GROUP`), `__init__.py` (thin re-export facade) +- `pyproject.toml` — `artifact-parser[dbt]` runtime dependency +- `docs/dbdocs-demo.yml` — the documented `health:` block (all thresholds + every rule name under `disable`) + default `run_results` +- `tests/fixtures/jaffle_shop/run_results.json` — sanitized plain-dbt run (29 tests) whose ids match the committed manifest (co-located with the artifacts so the default `/run_results.json` resolves) +- `dbdocs/site/builder.py` — `def _resolve_run_results_path`, `def build_data` (health key, `config=config.health`) +- `dbdocs/site/bundle/assets/js/data.js` — `normalize()` health default (`dimensions`/`testResults`/`note`) +- `dbdocs/site/bundle/assets/js/service.js` — `healthDimensions`, `healthEnabled` (issues>0), `healthTotalIssues`, `testResultsForNode` (data/unit split) +- `dbdocs/site/bundle/assets/js/ui.js` — `renderHealth`, `healthScorecard`, `healthDimensionSection`, `nodeTestResults` (model-page Tests) diff --git a/.claude/settings.json b/.claude/settings.json index 79e27ee..06d3c63 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -54,5 +54,8 @@ "statusLine": { "type": "command", "command": "input=$(cat); in=$(echo \"$input\" | jq -r '.context_window.total_input_tokens // 0'); out=$(echo \"$input\" | jq -r '.context_window.total_output_tokens // 0'); cr=$(echo \"$input\" | jq -r '.context_window.current_usage.cache_read_input_tokens // 0'); cw=$(echo \"$input\" | jq -r '.context_window.current_usage.cache_creation_input_tokens // 0'); cost=$(echo \"$in $out\" | awk '{printf \"%.4f\", ($1/1000000)*3 + ($2/1000000)*15}'); total=$((in + out)); printf \"tok:%s (in:%s out:%s) cache r:%s w:%s | $%s\" \"$total\" \"$in\" \"$out\" \"$cr\" \"$cw\" \"$cost\"" + }, + "enabledPlugins": { + "frontend-design@claude-plugins-official": true } } diff --git a/.claude/skills/dbdocs-code-review/SKILL.md b/.claude/skills/dbdocs-code-review/SKILL.md new file mode 100644 index 0000000..495afbf --- /dev/null +++ b/.claude/skills/dbdocs-code-review/SKILL.md @@ -0,0 +1,310 @@ +--- +name: dbdocs-code-review +description: Use when reviewing dbdocs changes (a diff, branch, or PR) for consistency, Python pluggability, design-pattern alignment, the 3-tier bundle-JS contract, complexity/scale, and the generated-SPA UX & accessibility. Produces a severity-graded findings report scoped to this codebase's documented patterns — not a generic lint. +--- + +# Reviewing dbdocs changes + +This is **dbdocs**' opinionated review: it checks a change against *this* +codebase's load-bearing patterns, not generic style. The six review dimensions +below are the contract. Read the change, then report findings under each +dimension, severity-graded, with `file:line` evidence. + +## Scope the change first + +Determine what's under review and read only the relevant tiers: + +```bash +git diff --stat # what changed +git status --short # incl. untracked new files +``` + +**Consistency**, **Design patterns**, and **Complexity / scale** apply to *every* +change — Python, bundle JS, CSS, config, docs, tests (Complexity is scoped to the +generate path). The other three dimensions are domain-scoped: + +- **Pluggable (Python modules)** → Python changes (`dbdocs/**/*.py`). +- **3-tier bundle JS** → bundle changes (`dbdocs/site/bundle/assets/{js,css}/**`). +- **UX & accessibility** → anything affecting the *rendered* SPA — bundle + JS/CSS/HTML (`dbdocs/site/bundle/**`) and the React Flow graph (`frontend/**`). +- `frontend/**` (React Flow graph) is the *compiled* graph app, not the 3-tier + shell — review it as its own app, not against the shell tiers, but it is still + subject to **Consistency**, **Design patterns**, **Complexity / scale**, and + **UX & accessibility** (e.g. the committed `assets/graph/` bundle must be + rebuilt when the source changes, and DAG/ERD work must respect windowing). + +Always run all six dimension checks; report "none" for a dimension with no +findings rather than skipping it. + +**Authority:** `.claude/design_patterns.md` (patterns + `file:line` citations) and +`.claude/skills/spa-site/SKILL.md` (the SPA/data-dict contract). The cited +**symbol** is authoritative; line numbers drift — grep the symbol. + +## Run the gates (report, don't fix) + +```bash +uv run ruff format --check . && uv run ruff check . +uv run ruff check --select PLC0415 . # no inline imports +uv run pytest --cov=dbdocs --cov-report=term-missing --cov-fail-under=100 +``` + +A green baseline is table stakes; the review is about what the gates *don't* +catch. Never fix failures during a review unless the user asks. + +## Dimension 1 — Consistency + +Does the change look like the code around it, and reuse what already exists? + +- **One class per file** (exception: multiple exception classes may share a file); + **no nested** functions/classes; **all imports at module top** (`ruff PLC0415`); + **no relative imports**. +- **Specific exception types** in `try/except` — never bare `except:` / + `except Exception`. Fail-soft paths log via the singleton `logger` + (`dbdocs.core.log`), never a new logger. +- **DRY**: repeated logic should be shared, not copy-pasted (helpers, fixtures via + `tests/conftest.py`). Flag duplicated blocks and divergent values for the *same* + concept (e.g. two CSS colors for one status, two constants for one threshold). +- **No backward-compat shims** unless explicitly asked. +- **Naming + docstrings match reality**: flag stale references (e.g. a docstring + naming a renamed/moved module), unused parameters, dead exports. +- **No duplicated lookups**: a value computed twice in one function (especially a + `service` accessor called repeatedly) should be cached in a local. + +## Dimension 2 — Pluggable (Python modules) + +New Python surface should extend an established seam, not invent a parallel one. + +- **Pipeline-stage layout**: `core/` → `extract/` → `site/`, one-way. `extract/` + and `site/` import from `core/`, never the reverse. A new extractor lives in + `dbdocs/extract/`; a feature spanning >2 modules becomes a **sub-package** + (`extract//`) with a **thin `__init__.py` facade** (re-export only — + no implementation) and the impl in named modules (e.g. `registry.py`, + `dimensions/`). Cross-module imports use **public** names (no leading-underscore + symbols imported across module boundaries). +- **Registry / plugin seams** (e.g. the health rule engine): a feature with a set + of interchangeable units should expose a registration API + (`register_rule`-style decorator/call), discover plugins via the + **`dbdocs.health_rules` entry-point group** and/or a config `rules_module` + dotted path, and keep the built-in baseline restorable (`reset_rules`). Check: + registration is **idempotent**; the registry is reset between tests (an autouse + conftest fixture); thresholds/behaviour are **config-driven** (read from the + `dbdocs.yml` block via the object, not hardcoded at the call site); a bad plugin + is **fail-soft** (caught, logged, skipped — never sinks `generate`). +- **Config**: new knobs are fields on `DbDocsConfig` loaded from `dbdocs.yml`; + build-control fields (not display metadata) go in `_NON_METADATA_FIELDS`; a CLI + override writes back to the config object before `ReportBuilder` runs (the + `--dialect` pattern). Document the knob in `dbdocs.yml.example`. + +## Dimension 3 — Align with design patterns + +**Applies to every change** (Python, SPA, CSS, config, docs, tests). +Cross-check the change against the full catalogue in `.claude/design_patterns.md` +— not just the patterns below. For each pattern the change touches, confirm it +**extends** the pattern rather than forking it. The patterns span all tiers: +the pipeline-stage layout, the config object, the data dict, artifact loading, +column lineage, graph rendering, the SPA modules, versioned deploy, the click +group, the singleton logger, and the always-built health section. Representative +checks: + +- **One data dict + external gzip payload** — new SPA surface extends + `ReportBuilder.build_data()`'s single dict and is read in the SPA; no second + hand-off channel, no re-inlining the payload into `index.html`. +- **Centralized artifact loading + the `schema_` gotcha** — artifact access goes + through `dbdocs/core/artifacts.py`; read `schema_` (the alias), never `.schema` + (a bound method) off a parsed node. +- **Manifest-base columns, catalog-enriched** (`extract/nodes.py`) — columns come + from the manifest (manifest order), catalog only *enriches* type + case-insensitively; a stale/empty catalog must not drop documented columns. +- **Fail-soft, parallel column-level lineage** (`extract/column_lineage.py`) — + per-model failures caught/logged/skipped; scope built once per model (not per + column); parallel above `_PARALLEL_THRESHOLD` via picklable work tuples (never + the Pydantic manifest across the process boundary). +- **Windowed graph rendering** (`frontend/`, committed `assets/graph/` bundle) — + the DAG windows to a focal neighborhood before layout; over + `MAX_UNFOCUSED_DAG_NODES` it shows a placeholder. A graph-UI change must rebuild + + commit the bundle (`task frontend:build`). +- **Bundled SPA directory resolution** (`site/builder.py`) — `BUNDLE_DIR` resolves + from `__file__`; `generate()` does `rmtree` + `copytree` for a clean build. + Anything new under `bundle/` must be covered by the `pyproject.toml` artifacts + glob (`dbdocs/site/bundle/**/*`) or it won't ship in the wheel. +- **Versioned deploy without mike** (`site/deploy.py`) — plain `/` dirs + + `versions.json`; segment validation against `^[A-Za-z0-9._-]+$`. Don't + reintroduce mike/external tooling. +- **Optional/always-built artifact-derived section** (Health Check) — fail-soft to + an empty-but-enabled section when the artifact is absent; parse with the right + library (`artifact-parser` for run_results, not the manifest parser); status + enums read via `.value`. +- **If the change adds or removes a load-bearing pattern**, `.claude/design_patterns.md` + must be updated in the same change (new entry + TOC + a concrete `file:line` + citation). Flag a missing/stale doc update as a finding. + +## Dimension 4 — 3-tier bundle JS + +The bundled shell SPA (`dbdocs/site/bundle/assets/js/`) is native ES modules in a +strict one-way `ui → service → data`. Verify: + +- **`data.js`** loads + `normalize()`s the payload, **defaulting every top-level + key** so the upper tiers never null-check the dict shape. +- **`service.js` is DOM-free** — pure domain logic over the normalized dict. No + `document`, no `el()`, no DOM reads/writes. Domain derivations (filtering, + grouping, scoring, lookups) belong here, exposed as accessors. +- **`ui.js` is the only DOM toucher** — it renders via `el()` and reads from + `service`. Flag domain logic that leaked into `ui` (it belongs in `service`) and + any DOM access that leaked into `service`. +- **Producer ↔ consumer in sync**: a Python data-dict change must have the SPA + side updated to read it (and vice-versa). Vendored UMD libs and the React Flow + graph bundle stay classic scripts setting globals (`window.dbdocsGraph`, + `MiniSearch`, `marked`); `data.js` re-exposes the payload on `window.dbdocsData` + for the graph bundle. +- **CSS** lives under `assets/css/`; the same status/concept uses **one** color + across pills/badges (don't introduce a divergent value). + +## Dimension 5 — Complexity / scale (the 1000s-of-models invariant) + +**Applies to every change that touches the generate path** (Python *and* the +graph bundle). dbdocs must stay performant on **3000+ model** projects — this is a +load-bearing product promise (parallel column lineage + windowed graph rendering +exist for exactly this). For each new or changed loop, recursion, or data +structure, state its **complexity in terms of N = #nodes (models+sources+tests)** +and flag anything worse than the budget below. + +What to compute and check: + +- **Annotate the hot path.** For each new function on the generate path, give its + Big-O in N. The bar: **whole-pass work should be ~`O(N)` or `O(N·k)`** for a + small constant/config `k` (e.g. #rules, #columns-per-model). Flag: + - **`O(N²)`** — a nested loop over all nodes (e.g. "for each model, scan all + models/edges"). Almost always avoidable by **indexing once** (build a + `dict`/`set` adjacency up front, then O(1) lookups — see + `ManifestGraph._build_adjacency`). The health rules are `O(N·rules)` with O(1) + `parents()`/`children()` for this reason. + - **Repeated recompute** — the same derived value computed per-iteration instead + of **memoized once** (e.g. `non_physical_chain_depth` caches on + `_chain_cache`; column-lineage builds `scope` once per model, not per column). + - **Re-parsing / re-scanning** — re-reading an artifact, re-parsing SQL, or + re-scanning entry points inside a loop. +- **Recursion depth.** dbt DAGs can be **thousands deep**. Any recursion over the + graph must be **iterative or memoized** — naive recursion hits Python's + ~1000-frame limit and `RecursionError`s `generate` on a deep chain. Prefer an + explicit stack + a visited set (cycle guard) over `def f(): ... f(parent)`. +- **Parallelism threshold.** CPU-bound per-model work (sqlglot parsing) fans out + across a `ProcessPoolExecutor` above `_PARALLEL_THRESHOLD`; only **picklable** + work tuples cross the boundary (never the Pydantic manifest/catalog). A new + heavy per-model step should follow this, not run a serial loop. +- **Payload size.** New data-dict keys are `O(N)` per node at worst — don't emit + `O(N²)` data (e.g. a full adjacency matrix). The payload is gzipped + external; + keep per-node records lean. +- **Graph bundle.** New DAG/ERD work must respect windowing — layout only the kept + set (`buildDagFlow(data, keepIds)`), never lay out all N nodes unfocused. +- **Fail-soft on blow-up.** A pathological input (deep cycle, huge fanout) must be + caught per-unit and skipped, not crash `generate` (the per-rule `try/except` in + `DimensionAnalyzer.analyze` catches `RecursionError` too). + +When reviewing, **prove it**: if a change adds a graph walk or per-model loop and +you're unsure, sketch a worst-case input (a 3000-deep view chain, a 3000-wide +fanout) and reason about whether it stays linear / terminates. Cite the offending +`file:line` and the input that breaks it. + +## Dimension 6 — UX & accessibility (the rendered SPA) + +**Scoped to changes that affect the *rendered* product** — the bundled shell SPA +(`dbdocs/site/bundle/**`: `js/`, `css/`, `index.html`) and the React Flow graph +(`frontend/**`). dbdocs ships a documentation site people *use*; a change can be +correct, pattern-aligned, and `O(N)` yet still ship a page that's unusable by +keyboard, unreadable on a phone, or silently broken on an empty/error state. +Report "none" for a change that doesn't touch the rendered SPA. Cite `file:line` +and, where it matters, the device/AT (assistive tech) or state that exposes it. + +### Accessibility + +- **Keyboard reachable + operable.** Every interactive affordance is a real + control (``, ` +
diff --git a/docs/dbdocs-demo.yml b/docs/dbdocs-demo.yml index 3f721f3..25d2937 100644 --- a/docs/dbdocs-demo.yml +++ b/docs/dbdocs-demo.yml @@ -24,6 +24,64 @@ readme: README.md # mike publishes it at /latest/demo/latest/. This path is gitignored (/docs/demo). output_dir: docs/demo +# The Health Check page is always built from the manifest (the six DPE +# dimensions). The per-test pass/fail detail comes from run_results.json, which +# lives alongside the artifacts in target_dir, so the default +# /run_results.json resolves — no explicit run_results needed. +# +# The health rule engine is configurable. Every threshold below is set to its +# dbt-project-evaluator default; tune them, disable individual rules or whole +# dimensions, or load custom rules. All built-in rules are listed (commented) +# under `disable` for reference. +health: + # Rule thresholds (DPE defaults shown). + thresholds: + model_fanout: 3 # > N direct model children is flagged + too_many_joins: 7 # >= N upstream dependencies is flagged + chained_view_dependencies: 4 # >= N-deep view/ephemeral chain is flagged + + # Disable individual rules by name (none disabled in the demo). The full set: + disable: [] + # testing: + # - test_coverage + # - missing_primary_key_tests + # modeling: + # - direct_join_to_source + # - duplicate_sources + # - model_fanout + # - multiple_sources_joined + # - rejoining_of_upstream_concepts + # - root_models + # - source_fanout + # - staging_dependent_on_staging + # - staging_dependent_on_marts_or_intermediate + # - unused_sources + # - too_many_joins + # documentation: + # - undocumented_models + # - undocumented_sources + # - undocumented_source_tables + # structure: + # - model_naming_conventions + # - model_directories + # - source_directories + # performance: + # - chained_view_dependencies + # - exposure_parents_materializations + # governance: + # - public_models_without_contracts + # - undocumented_public_models + # - exposures_dependent_on_private_models + + # Disable whole dimensions (none in the demo). One of: testing, modeling, + # documentation, structure, performance, governance. + disable_dimensions: [] + + # Load custom rules from a dotted module path; the module calls + # `register_rule("", fn)` at import time. Installed packages can also + # ship rules via the `dbdocs.health_rules` entry-point group. + # rules_module: my_project.dbdocs_health_rules + # dbterd configs dbterd: algo: model_contract diff --git a/docs/index.md b/docs/index.md index 404048f..099bad7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,9 +4,9 @@ dbdocs logo

-

An alternative dbt docs site — catalog + ERD + column-level lineage.

+

An alternative dbt docs site — catalog + ERD + column-level lineage + versioned deploys, all in one CLI.

-Turn your dbt artifacts into a single self-contained `index.html`: a browsable catalog, an interactive lineage DAG and ERD, and **column-level lineage** derived straight from your compiled SQL. No server, no database, no build step — just a file you can open or host anywhere. +Turn your dbt artifacts into a self-contained docs site: a browsable catalog, an entity-relationship diagram, an interactive lineage DAG, and **column-level lineage** traced from your compiled SQL — all in one `dbdocs generate`. No server, no database, no build step. Serve it with `dbdocs serve`, or deploy versioned builds anywhere a static host will take them. [:rocket: Try the live demo](/latest/demo/latest/){ .md-button .md-button--primary target="_blank" } [Quickstart](./nav/guide/quickstart.md){ .md-button } @@ -34,14 +34,46 @@ the catalog, the lineage DAG, the ERD, and column-level lineage. ## What you get -dbt's own docs are great until you want lineage at the *column* level — that's the gap dbdocs fills. +dbt's built-in docs are great — right up until you want to know *which upstream column fed this downstream column*, or *which tables relate to each other*, or *what changed between last week's docs and today's*. dbdocs fills all three gaps without asking you to install a documentation framework or maintain a separate ERD tool. -- **Catalog navigation** grouped by database and schema, with client-side search (no backend). +### ERD + column-level lineage, side by side + +The entity-relationship diagram (powered by [dbterd](https://github.com/datnguye/dbterd)) shows table relationships; column lineage (traced by [sqlglot](https://github.com/tobymao/sqlglot) from compiled SQL) shows exactly which column fed which. Most alternatives give you one or the other — dbdocs gives you both, in the same site, from the same artifacts. + +### Column impact analysis + +Select any column and see its downstream dependents across the project. Know what a schema change will break before you run it — not after. + +### Deep-link URLs + +Every focused node, column, and filtered DAG view has a shareable URL. Paste it in Slack and your teammate lands on exactly the right model, column, or graph state — no "go to the DAG and find orders and then…" required. + +### Any sqlglot-supported dialect + +The dialect for column-lineage parsing is auto-detected from your manifest's `adapter_type` (Snowflake, BigQuery, Redshift, DuckDB, PostgreSQL, Databricks/Spark, Trino, and more — anything [sqlglot](https://github.com/tobymao/sqlglot) understands). Override it per-project with `dialect:` in `dbdocs.yml` when auto-detection isn't enough. + +### Scales without freezing + +Column-lineage parsing fans out across CPU cores automatically above 500 models, so large projects finish in roughly the same wall-clock time as small ones. The React Flow DAG is windowed, so a 1 000-model graph doesn't turn your browser into a space heater. The data payload ships as an external gzip (`dbdocs-data.json.gz`, decompressed client-side by the browser's native `DecompressionStream`) — `index.html` stays tiny regardless of project size. + +### Fail-soft + +One model with SQL sqlglot can't parse gets logged and skipped. It never sinks the whole generate run, so a single dialect quirk or macro-heavy model doesn't block you from seeing the rest of your project. + +### Project Health Check + +The SPA always includes a Health Check page, built from the `run_results.json` that a `dbt build`/`dbt test` produced (default: `/run_results.json`, override with `--run-results`). Every dbt test becomes a finding, grouped by what it checks — integrity (not-null/unique), referential (relationships), validity (accepted values), business logic (expressions), and freshness. The test type, tested model, and column come from your `manifest.json`. dbdocs only reads the static artifact; it never runs dbt and never touches your warehouse, so the page reflects exactly your last build. Missing or malformed `run_results.json`? Fail-soft: the page simply stays empty and a warning is logged, not a stack trace. + +### Versioned deploys, no plugins + +`dbdocs deploy --version v1.2 --alias latest` generates into a plain directory tree, writes a `versions.json` index, and the SPA renders a version dropdown. No mike, no external tooling, no surprise dependencies — any static host can serve the output. + +### Everything else you'd expect + +- **Catalog navigation** grouped by database and schema, with client-side full-text search (no backend). - **Per-model detail** — columns (type / tags / description), compiled and raw SQL, and the macros each model resolves. -- **Interactive graphs** — a node-level lineage DAG and an ERD, both built on [React Flow](https://reactflow.dev/): pan / zoom / minimap, automatic [dagre](https://github.com/dagrejs/dagre) layout, filter-and-focus, and deep-links to any node. -- **Column-level lineage** traced from each model's compiled SQL via [sqlglot](https://github.com/tobymao/sqlglot). +- **Interactive graphs** — both the lineage DAG and the ERD are built on [React Flow](https://reactflow.dev/): pan / zoom / minimap, automatic [dagre](https://github.com/dagrejs/dagre) layout, and filter-and-focus. - **Dark / light** theme. -- **Versioned deploy** with a built-in version dropdown — no mike, no plugins. --- @@ -76,10 +108,14 @@ dbt docs generate # writes target/manifest.json + target/catalog.json Then generate, serve, and open the site: ```bash -dbdocs generate # builds ./site/index.html with all data baked in +dbdocs generate # builds ./site/ with index.html + dbdocs-data.json.gz dbdocs serve # static http server on http://127.0.0.1:8000 ``` +The site must be served over HTTP (not opened as a local file) because it fetches +the data payload at load time. `dbdocs serve` handles that locally; any static host +works for deployment. + Head to the [Quickstart guide](./nav/guide/quickstart.md) for the full walkthrough. --- diff --git a/docs/plans/scale-mobile-e2e-branding.md b/docs/plans/scale-mobile-e2e-branding.md deleted file mode 100644 index 194871e..0000000 --- a/docs/plans/scale-mobile-e2e-branding.md +++ /dev/null @@ -1,284 +0,0 @@ -# Plan: scale, mobile, E2E tests, branding - -Six workstreams for `dbdocs`. Each is self-contained and can ship on its own -branch/PR. Workstream 0 is a no-behavior-change refactor sequenced first so the -feature workstreams land on tidy seams; 1–5 are independent. Ordered by leverage. -Every Python change stays inside the `core/` → `extract/` → `site/` pipeline -pattern and keeps the 100% coverage gate. - -## Table of contents - -- [Plan: scale, mobile, E2E tests, branding](#plan-scale-mobile-e2e-tests-branding) - - [Table of contents](#table-of-contents) - - [0. SPA 3-tier refactor (data → service → ui)](#0-spa-3-tier-refactor-data--service--ui) - - [1. Standalone versioned demo via `dbdocs deploy`](#1-standalone-versioned-demo-via-dbdocs-deploy) - - [2. Customizable logo + favicon](#2-customizable-logo--favicon) - - [3. Performance at 3000+ models](#3-performance-at-3000-models) - - [4. Mobile / responsive UI](#4-mobile--responsive-ui) - - [5. Playwright E2E tests (TypeScript)](#5-playwright-e2e-tests-typescript) - - [Agentic-docs updates](#agentic-docs-updates) - ---- - -## 0. SPA 3-tier refactor (data → service → ui) - -A **pure restructure, no behavior change** — done first so the feature -workstreams below each edit one tier instead of hunting through a 500-line IIFE, -and so the Playwright E2E suite (workstream 5) asserts against stable, named -seams. Ship as its own PR before the performance payload work (workstream 3a). - -**Why now.** `app.js` is today a single ~504-line IIFE -(`(function () { … })()`) where all three concerns are present but interleaved: -- **data** — `var DATA = window.dbdocsData || { … }` (`app.js:8`). -- **service** — pure derivations over `DATA` with no DOM: `shortName`, - `footerMeta`, `parseHash`, `resourceCards`, `repoUrl`, `pluralize`, counts. -- **ui** — DOM only: `el`, `clear`, `route`, `buildNav`, `render*`, - `graphMount`. - -The seams already exist; this names them. Three queued workstreams poke exactly -these seams — the gzipped payload (3a) is a **data**-tier change, logo/favicon (2) -is a **service→ui** metadata change, the mobile drawer (4) is **pure ui** — so -splitting first de-risks all of them. - -**Hard constraint — no build step.** The shell bundle is committed vanilla JS; -`dbdocs generate` is "pure-Python and build-step-free — it just copies the -prebuilt bundle." So the tiering uses **native ES modules** (`