diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2ee336a..833af30 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,6 +10,7 @@ - [ ] Plugin installs standalone (`uv pip install -e plugins/`) - [ ] Plugin docs regenerated if plugin docs, list, or metadata changed (`make plugin-docs`) - [ ] Documentation builds if docs changed (`make docs`) +- [ ] `catalog/plugins.json` regenerated if plugin list or metadata changed (`make catalog`) - [ ] `.github/CODEOWNERS` regenerated if ownership changed (`make codeowners`) - [ ] Per-plugin `CODEOWNERS` file included (auto-created by `ddp new`) - [ ] NVIDIA SPDX headers on all files (`make check-license-headers`) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f55a23d..b6febe6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: with: python-version: "3.10" enable-cache: true - # test uses isolated venvs, not workspace sync + - run: make sync - run: make test validate: @@ -60,5 +60,6 @@ jobs: path: | docs/plugins/** zensical.toml + catalog/plugins.json.new .github/CODEOWNERS.new if-no-files-found: ignore diff --git a/AGENTS.md b/AGENTS.md index 22b9336..9ab2ad3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,5 +198,5 @@ Release facts: - [DataDesigner GitHub](https://github.com/NVIDIA-NeMo/DataDesigner) - [DataDesigner Latest Release Notes](https://github.com/NVIDIA-NeMo/DataDesigner/releases/latest) - [DataDesigner Plugin Authoring Guide](https://nvidia-nemo.github.io/DataDesigner/latest/plugins/overview/) -- [data-designer-plugins authoring guide](docs/adding-a-plugin.md) -- [data-designer-plugins plugin catalog](docs/catalog.md) +- [data-designer-plugins authoring guide](docs/authoring.md) +- [data-designer-plugins plugin catalog](catalog/plugins.json) diff --git a/Makefile b/Makefile index ba00d36..772abe9 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -.PHONY: sync lint format test validate docs docs-server plugin-docs check-plugin-docs codeowners check-codeowners check-license-headers update-license-headers check all bump release build-plugin validate-release test-plugin check-owner +.PHONY: sync lint format test test-devtools test-plugins validate docs docs-server plugin-docs catalog check-plugin-docs check-catalog codeowners check-codeowners check-license-headers update-license-headers check all bump release build-plugin validate-release test-plugin check-owner # ── Setup ──────────────────────────────────────────────────────────────── @@ -22,7 +22,12 @@ format: # Auto-discover plugins and test each in an isolated venv. # Catches dependency leaks that workspace-level testing misses. -test: +test: test-devtools test-plugins + +test-devtools: + uv run pytest devtools/ddp/tests/ -v + +test-plugins: @failed=0; \ for pyproject in plugins/*/pyproject.toml; do \ plugin_dir="$$(dirname "$$pyproject")"; \ @@ -57,17 +62,23 @@ DOCS_DEV_ADDR ?= localhost:8000 docs-server: plugin-docs uv run zensical serve --dev-addr $(DOCS_DEV_ADDR) -# ── Plugin docs & CODEOWNERS ────────────────────────────────────────────── +# ── Plugin docs, catalog & CODEOWNERS ───────────────────────────────────── plugin-docs: uv run ddp plugin-docs +catalog: + uv run ddp sync catalog + codeowners: uv run ddp codeowners > .github/CODEOWNERS check-plugin-docs: uv run ddp plugin-docs --check +check-catalog: + uv run ddp sync catalog --check + check-codeowners: uv run ddp codeowners > .github/CODEOWNERS.new diff .github/CODEOWNERS .github/CODEOWNERS.new @@ -81,7 +92,7 @@ update-license-headers: # ── Aggregate targets ──────────────────────────────────────────────────── -check: check-plugin-docs check-codeowners check-license-headers +check: check-plugin-docs check-catalog check-codeowners check-license-headers all: lint test validate check docs diff --git a/README.md b/README.md index 63e27e4..8864184 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This generates a complete plugin skeleton under `plugins/data-designer-my-plugin DataDesignerPlugins/ |-- devtools/ | `-- ddp/ # Monorepo management tooling (ddp CLI, dev-only) +|-- catalog/ # Machine-consumable plugin catalog data |-- plugins/ # One directory per plugin (auto-discovered by uv) | `-- data-designer-template/ # Reference implementation `-- docs/ # Zensical documentation source @@ -43,8 +44,9 @@ make lint # Lint and format check (ruff) make format # Auto-fix lint issues and reformat make test # Test each plugin in an isolated venv make validate # Run assert_valid_plugin on all entry points -make check # Verify generated plugin docs, CODEOWNERS, and license headers are up to date +make check # Verify generated plugin docs, catalog, CODEOWNERS, and license headers are up to date make plugin-docs # Regenerate docs/plugins/ from per-plugin docs and metadata +make catalog # Regenerate catalog/plugins.json make docs # Build the Zensical documentation site make docs-server # Serve docs locally at http://localhost:8000 make all # lint + test + validate + check + docs (full local CI) @@ -60,6 +62,7 @@ If you change plugin docs, plugin metadata, or ownership, regenerate derived fil ```bash make plugin-docs # Regenerate plugin documentation site inputs +make catalog # Regenerate catalog/plugins.json make codeowners # Regenerate CODEOWNERS make update-license-headers # Fix SPDX headers ``` @@ -71,6 +74,7 @@ The `ddp` command manages the monorepo. Run `uv run ddp --help` to see all subco | Command | Description | |---------|-------------| | `ddp new ` | Scaffold a new plugin | +| `ddp sync catalog` | Sync the static plugin catalog JSON | | `ddp validate` | Validate all installed plugins | | `ddp plugin-docs` | Generate plugin docs site inputs | | `ddp codeowners` | Aggregate CODEOWNERS to stdout | diff --git a/catalog/plugins.json b/catalog/plugins.json new file mode 100644 index 0000000..bab1a70 --- /dev/null +++ b/catalog/plugins.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "plugins": [ + { + "name": "text-transform", + "plugin_type": "column-generator", + "description": "Template Data Designer plugin \u2014 text transform column generator", + "package": { + "name": "data-designer-template", + "version": "0.1.0", + "path": "plugins/data-designer-template" + }, + "entry_point": { + "group": "data_designer.plugins", + "name": "text-transform", + "value": "data_designer_template.plugin:plugin" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + } + } + ] +} diff --git a/devtools/ddp/pyproject.toml b/devtools/ddp/pyproject.toml index 181da3a..84fe79d 100644 --- a/devtools/ddp/pyproject.toml +++ b/devtools/ddp/pyproject.toml @@ -7,6 +7,7 @@ version = "0.1.0" description = "Local dev-only monorepo management tooling for data-designer-plugins (not a runtime dependency)" requires-python = ">=3.10" dependencies = [ + "packaging>=24.0", # Backport of the stdlib `tomllib` (added in 3.11); only installed on 3.10. "tomli>=1.1.0; python_version < '3.11'", ] diff --git a/devtools/ddp/src/ddp/__init__.py b/devtools/ddp/src/ddp/__init__.py index a2aa58e..f9552fa 100644 --- a/devtools/ddp/src/ddp/__init__.py +++ b/devtools/ddp/src/ddp/__init__.py @@ -4,6 +4,6 @@ """Local dev-only monorepo management tooling for data-designer-plugins. This package is NOT a runtime dependency of any plugin. It provides CLI -tools for scaffolding, validation, plugin docs generation, CODEOWNERS -aggregation, and license-header management across the monorepo. +tools for scaffolding, validation, plugin docs generation, catalog syncing, +CODEOWNERS aggregation, and license-header management across the monorepo. """ diff --git a/devtools/ddp/src/ddp/catalog.py b/devtools/ddp/src/ddp/catalog.py new file mode 100644 index 0000000..7eb98ce --- /dev/null +++ b/devtools/ddp/src/ddp/catalog.py @@ -0,0 +1,682 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sync a JSON plugin catalog from package metadata and plugin objects.""" + +from __future__ import annotations + +import importlib.metadata +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlparse + +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version + +from ddp._repo import find_repo_root, load_toml + +CATALOG_SCHEMA_VERSION = 1 +REPO_ROOT = find_repo_root() +PLUGINS_DIR = REPO_ROOT / "plugins" +CATALOG_BASE_PATH = REPO_ROOT / "catalog" +PLUGINS_CATALOG_FILENAME = "plugins.json" +PLUGINS_CATALOG_PATH = CATALOG_BASE_PATH / PLUGINS_CATALOG_FILENAME +DATA_DESIGNER_DISTRIBUTION_NAME = "data-designer" +PLUGIN_ENTRY_POINT_GROUP = "data_designer.plugins" + + +class CatalogError(RuntimeError): + """Raised when a catalog entry cannot be generated.""" + + +@dataclass(frozen=True) +class CatalogEntry: + """One plugin entry in the JSON catalog. + + Attributes: + plugin_package: Python package name from ``[project].name``. + version: Package version from ``[project].version``. + name: Runtime DataDesigner plugin name. + plugin_type: Runtime DataDesigner plugin type value. + description: Package description from ``[project].description``. + entry_point_name: Entry point name in the ``data_designer.plugins`` group. + entry_point_value: Import target registered for the entry point. + repository_path: Path to the plugin package from the repository root. + python_requires: Python version specifier from ``[project].requires-python``. + data_designer_requirement: Direct ``data-designer`` dependency + requirement string. + data_designer_version_specifier: Version specifier from the package's + direct ``data-designer`` dependency. + data_designer_marker: Environment marker from the package's direct + ``data-designer`` dependency, or ``None`` when the requirement is + unconditionally active. + """ + + plugin_package: str + version: str + name: str + plugin_type: str + description: str + entry_point_name: str + entry_point_value: str + repository_path: str + python_requires: str + data_designer_requirement: str + data_designer_version_specifier: str + data_designer_marker: str | None + + +def main() -> None: + """Generate a JSON catalog of all plugin entry points and print to stdout.""" + try: + entries = discover_catalog_entries(PLUGINS_DIR) + except CatalogError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + + print(render_catalog_json(entries), end="") + + +def sync_catalog() -> Path: + """Write the repository plugin catalog JSON file. + + Returns: + Absolute path to the synced catalog file. + + Raises: + CatalogError: If a catalog entry cannot be generated. + """ + entries = discover_catalog_entries(PLUGINS_DIR) + PLUGINS_CATALOG_PATH.parent.mkdir(parents=True, exist_ok=True) + PLUGINS_CATALOG_PATH.write_text(render_catalog_json(entries), encoding="utf-8") + return PLUGINS_CATALOG_PATH + + +def check_catalog() -> bool: + """Check whether the repository plugin catalog JSON file is current. + + When the catalog is stale, a sibling ``.new`` file is written with the + expected content so CI can upload it as a drift artifact. + + Returns: + ``True`` when the catalog is current, otherwise ``False``. + + Raises: + CatalogError: If a catalog entry cannot be generated. + """ + entries = discover_catalog_entries(PLUGINS_DIR) + expected = render_catalog_json(entries) + new_path = PLUGINS_CATALOG_PATH.with_name(f"{PLUGINS_CATALOG_PATH.name}.new") + if PLUGINS_CATALOG_PATH.exists() and PLUGINS_CATALOG_PATH.read_text(encoding="utf-8") == expected: + new_path.unlink(missing_ok=True) + return True + + new_path.parent.mkdir(parents=True, exist_ok=True) + new_path.write_text(expected, encoding="utf-8") + return False + + +def discover_catalog_entries(plugins_dir: Path) -> list[CatalogEntry]: + """Discover catalog entries for local plugin packages. + + Args: + plugins_dir: Repository ``plugins/`` directory. + + Returns: + Entries sorted by package name, then runtime plugin name. + + Raises: + CatalogError: If a local entry point is not installed, cannot be loaded, + or does not load to a DataDesigner ``Plugin`` object. + """ + entries: list[CatalogEntry] = [] + for toml_path in sorted(plugins_dir.glob("*/pyproject.toml")): + data = load_toml(toml_path) + project = project_table_for_pyproject(data, toml_path) + + name = required_project_string(toml_path.parent.name, project, "name") + version = project_version(toml_path.parent.name, project.get("version")) + description = optional_project_string(name, project, "description") + python_requires = python_requires_specifier(name, project.get("requires-python")) + data_designer_requirement = data_designer_requirement_for_dependencies( + package_name=name, + dependencies=project.get("dependencies", []), + ) + data_designer = Requirement(data_designer_requirement) + data_designer_version_specifier = str(data_designer.specifier) + data_designer_marker = str(data_designer.marker) if data_designer.marker is not None else None + + entry_points = data_designer_entry_points(name, project) + repository_path = toml_path.parent.relative_to(plugins_dir.parent).as_posix() + for entry_point_name, entry_point_value in sorted(entry_points.items()): + entries.append( + catalog_entry_for_entry_point( + package_name=name, + version=version, + description=description, + entry_point_name=entry_point_name, + entry_point_value=entry_point_value, + package_dir=toml_path.parent, + repository_path=repository_path, + python_requires=python_requires, + data_designer_requirement=data_designer_requirement, + data_designer_version_specifier=data_designer_version_specifier, + data_designer_marker=data_designer_marker, + ) + ) + + return sorted(entries, key=lambda entry: (entry.plugin_package, entry.name)) + + +def catalog_entry_for_entry_point( + package_name: str, + version: str, + description: str, + entry_point_name: str, + entry_point_value: str, + package_dir: Path, + repository_path: str, + python_requires: str, + data_designer_requirement: str, + data_designer_version_specifier: str, + data_designer_marker: str | None, +) -> CatalogEntry: + """Build a catalog entry from an installed DataDesigner plugin entry point. + + Args: + package_name: Local plugin package name. + version: Local plugin package version. + description: Local plugin package description. + entry_point_name: Entry point name in the ``data_designer.plugins`` group. + entry_point_value: Import target registered for the entry point. + package_dir: Local plugin package directory. + repository_path: Path to the plugin package from the repository root. + python_requires: Python version specifier from + ``[project].requires-python``. + data_designer_requirement: Direct ``data-designer`` dependency + requirement string. + data_designer_version_specifier: Version specifier from the package's + direct ``data-designer`` dependency. + data_designer_marker: Environment marker from the package's direct + ``data-designer`` dependency, or ``None`` when the requirement is + unconditionally active. + + Returns: + Catalog entry with runtime plugin metadata. + + Raises: + CatalogError: If plugin metadata cannot be loaded or read. + """ + plugin = load_plugin_from_entry_point( + package_name=package_name, + entry_point_name=entry_point_name, + entry_point_value=entry_point_value, + package_dir=package_dir, + ) + try: + plugin_name = plugin.name + plugin_type = plugin.plugin_type.value + except Exception as exc: + raise CatalogError( + f"could not read runtime metadata for package {package_name!r} entry point {entry_point_name!r}: {exc}" + ) from exc + + if not isinstance(plugin_name, str) or not plugin_name: + raise CatalogError( + f"package {package_name!r} entry point {entry_point_name!r} has invalid plugin.name {plugin_name!r}" + ) + if not isinstance(plugin_type, str) or not plugin_type: + raise CatalogError( + f"package {package_name!r} entry point {entry_point_name!r} has invalid plugin.plugin_type.value " + f"{plugin_type!r}" + ) + + return CatalogEntry( + plugin_package=package_name, + version=version, + name=plugin_name, + plugin_type=plugin_type, + description=description, + entry_point_name=entry_point_name, + entry_point_value=entry_point_value, + repository_path=repository_path, + python_requires=python_requires, + data_designer_requirement=data_designer_requirement, + data_designer_version_specifier=data_designer_version_specifier, + data_designer_marker=data_designer_marker, + ) + + +def project_table_for_pyproject(data: dict[str, Any], toml_path: Path) -> dict[str, Any]: + """Return the validated ``[project]`` table from a plugin ``pyproject.toml``. + + Args: + data: Parsed ``pyproject.toml`` content. + toml_path: Path to the parsed ``pyproject.toml``. + + Returns: + The ``[project]`` table. + + Raises: + CatalogError: If the table is missing or malformed. + """ + project = data.get("project") + if not isinstance(project, dict): + raise CatalogError(f"package at {toml_path.parent.as_posix()!r} has invalid [project] table") + return project + + +def required_project_string(package_name: str, project: dict[str, Any], key: str) -> str: + """Return a required non-empty string value from ``[project]``. + + Args: + package_name: Local plugin package name or directory name. + project: Parsed ``[project]`` table. + key: Project metadata key. + + Returns: + Project metadata string value. + + Raises: + CatalogError: If the value is missing or not a non-empty string. + """ + value = project.get(key) + if not isinstance(value, str) or not value: + raise CatalogError(f"package {package_name!r} has invalid [project].{key}; expected a non-empty string") + return value + + +def optional_project_string(package_name: str, project: dict[str, Any], key: str) -> str: + """Return an optional string value from ``[project]``. + + Args: + package_name: Local plugin package name. + project: Parsed ``[project]`` table. + key: Project metadata key. + + Returns: + Project metadata string value, or ``""`` when omitted. + + Raises: + CatalogError: If the value is present but not a string. + """ + value = project.get(key, "") + if not isinstance(value, str): + raise CatalogError(f"package {package_name!r} has invalid [project].{key}; expected a string") + return value + + +def project_version(package_name: str, version: object) -> str: + """Return a validated PEP 440 package version. + + Args: + package_name: Local plugin package name or directory name. + version: Raw ``[project].version`` value. + + Returns: + Canonical PEP 440 package version. + + Raises: + CatalogError: If the version is missing, not a string, or not PEP 440. + """ + if not isinstance(version, str) or not version: + raise CatalogError(f"package {package_name!r} has invalid [project].version; expected a non-empty string") + try: + return str(Version(version)) + except InvalidVersion as exc: + raise CatalogError(f"package {package_name!r} has invalid [project].version {version!r}: {exc}") from exc + + +def python_requires_specifier(package_name: str, requires_python: object) -> str: + """Return a validated Python compatibility specifier. + + Args: + package_name: Local plugin package name. + requires_python: Raw ``[project].requires-python`` value. + + Returns: + Python version specifier string. + + Raises: + CatalogError: If the specifier is missing, malformed, or empty. + """ + if not isinstance(requires_python, str) or not requires_python: + raise CatalogError( + f"package {package_name!r} has invalid [project].requires-python; expected a non-empty string" + ) + try: + specifier = SpecifierSet(requires_python) + except InvalidSpecifier as exc: + raise CatalogError( + f"package {package_name!r} has invalid [project].requires-python {requires_python!r}: {exc}" + ) from exc + if not str(specifier): + raise CatalogError( + f"package {package_name!r} has invalid [project].requires-python; expected at least one version specifier" + ) + return str(specifier) + + +def data_designer_entry_points(package_name: str, project: dict[str, Any]) -> dict[str, str]: + """Return validated DataDesigner plugin entry points. + + Args: + package_name: Local plugin package name. + project: Parsed ``[project]`` table. + + Returns: + Entry point mapping from entry point names to import targets. + + Raises: + CatalogError: If the entry point table is missing, empty, or malformed. + """ + entry_points = project.get("entry-points") + if not isinstance(entry_points, dict): + raise CatalogError(f"package {package_name!r} must declare [project.entry-points.{PLUGIN_ENTRY_POINT_GROUP!r}]") + + plugin_entry_points = entry_points.get(PLUGIN_ENTRY_POINT_GROUP) + if not isinstance(plugin_entry_points, dict) or not plugin_entry_points: + raise CatalogError( + f"package {package_name!r} must declare at least one [project.entry-points.{PLUGIN_ENTRY_POINT_GROUP!r}]" + ) + + for entry_point_name, entry_point_value in plugin_entry_points.items(): + if not isinstance(entry_point_name, str) or not entry_point_name: + raise CatalogError( + f"package {package_name!r} has invalid {PLUGIN_ENTRY_POINT_GROUP!r} entry point name " + f"{entry_point_name!r}; expected a non-empty string" + ) + if not isinstance(entry_point_value, str) or not entry_point_value: + raise CatalogError( + f"package {package_name!r} entry point {entry_point_name!r} has invalid value " + f"{entry_point_value!r}; expected a non-empty string" + ) + + return plugin_entry_points + + +def data_designer_requirement_for_dependencies(package_name: str, dependencies: object) -> str: + """Return the direct DataDesigner dependency requirement for a package. + + Args: + package_name: Local plugin package name. + dependencies: Package dependency requirement strings from + ``[project].dependencies``. + + Returns: + Requirement string for the package's direct ``data-designer`` + dependency. The returned requirement must include a version specifier. + + Raises: + CatalogError: If dependencies are malformed, missing, or ambiguous. + """ + if not isinstance(dependencies, list) or not all(isinstance(dependency, str) for dependency in dependencies): + raise CatalogError(f"package {package_name!r} has invalid [project].dependencies; expected a list of strings") + + matching_requirements: list[str] = [] + for dependency in dependencies: + try: + requirement = Requirement(dependency) + except InvalidRequirement as exc: + raise CatalogError(f"package {package_name!r} has invalid dependency {dependency!r}: {exc}") from exc + + if canonicalize_name(requirement.name) == DATA_DESIGNER_DISTRIBUTION_NAME: + if not requirement.specifier: + raise CatalogError( + f"package {package_name!r} direct {DATA_DESIGNER_DISTRIBUTION_NAME!r} dependency " + "must include a version specifier" + ) + matching_requirements.append(dependency) + + if not matching_requirements: + raise CatalogError( + f"package {package_name!r} must declare a direct {DATA_DESIGNER_DISTRIBUTION_NAME!r} dependency " + "to publish catalog compatibility metadata" + ) + if len(matching_requirements) > 1: + raise CatalogError( + f"package {package_name!r} declares multiple direct {DATA_DESIGNER_DISTRIBUTION_NAME!r} dependencies" + ) + return matching_requirements[0] + + +def load_plugin_from_entry_point( + package_name: str, + entry_point_name: str, + entry_point_value: str, + package_dir: Path, +) -> Any: + """Load and validate an installed DataDesigner plugin entry point. + + Args: + package_name: Local plugin package name. + entry_point_name: Entry point name in the ``data_designer.plugins`` group. + entry_point_value: Expected import target from the local + ``pyproject.toml``. + package_dir: Local plugin package directory. + + Returns: + Loaded DataDesigner ``Plugin`` object. + + Raises: + CatalogError: If the entry point is missing, fails to load, or returns + a non-``Plugin`` object. + """ + try: + from data_designer.plugins.plugin import Plugin + except Exception as exc: + raise CatalogError( + f"could not import DataDesigner Plugin while loading package {package_name!r} " + f"entry point {entry_point_name!r}: {exc}" + ) from exc + + entry_point = find_installed_entry_point( + package_name=package_name, + entry_point_name=entry_point_name, + entry_point_value=entry_point_value, + package_dir=package_dir, + ) + try: + plugin = entry_point.load() + except Exception as exc: + raise CatalogError(f"could not load package {package_name!r} entry point {entry_point_name!r}: {exc}") from exc + + if not isinstance(plugin, Plugin): + raise CatalogError( + f"package {package_name!r} entry point {entry_point_name!r} loaded {type(plugin).__name__}, " + "expected data_designer.plugins.plugin.Plugin" + ) + return plugin + + +def find_installed_entry_point( + package_name: str, + entry_point_name: str, + entry_point_value: str, + package_dir: Path, +) -> importlib.metadata.EntryPoint: + """Find an installed entry point owned by a local package. + + Args: + package_name: Local plugin package name. + entry_point_name: Entry point name in the ``data_designer.plugins`` group. + entry_point_value: Expected import target from the local + ``pyproject.toml``. + package_dir: Local plugin package directory. + + Returns: + Matching installed entry point. + + Raises: + CatalogError: If no installed entry point matches the package and name. + """ + normalized_package_name = normalize_distribution_name(package_name) + for entry_point in importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP): + distribution_name = entry_point_distribution_name(entry_point) + if distribution_name is None: + continue + if ( + normalize_distribution_name(distribution_name) == normalized_package_name + and entry_point.name == entry_point_name + ): + validate_installed_entry_point( + package_name=package_name, + entry_point=entry_point, + entry_point_value=entry_point_value, + package_dir=package_dir, + ) + return entry_point + + raise CatalogError( + f"package {package_name!r} entry point {entry_point_name!r} is not installed; " + "run `make sync` before syncing the catalog" + ) + + +def validate_installed_entry_point( + package_name: str, + entry_point: importlib.metadata.EntryPoint, + entry_point_value: str, + package_dir: Path, +) -> None: + """Validate that an installed entry point matches local package metadata. + + Args: + package_name: Local plugin package name. + entry_point: Installed entry point selected by package and name. + entry_point_value: Expected import target from the local + ``pyproject.toml``. + package_dir: Local plugin package directory. + + Raises: + CatalogError: If the installed entry point target or source path does + not match the local plugin package. + """ + if entry_point.value != entry_point_value: + raise CatalogError( + f"package {package_name!r} entry point {entry_point.name!r} is stale; installed target " + f"{entry_point.value!r} does not match pyproject target {entry_point_value!r}. Run `make sync`." + ) + + source_path = entry_point_distribution_source_path(entry_point) + expected_path = package_dir.resolve() + if source_path != expected_path: + raise CatalogError( + f"package {package_name!r} entry point {entry_point.name!r} is installed from " + f"{source_path.as_posix() if source_path is not None else 'an unknown source'}, expected " + f"{expected_path.as_posix()}. Run `make sync`." + ) + + +def entry_point_distribution_source_path(entry_point: importlib.metadata.EntryPoint) -> Path | None: + """Return the editable source path for an installed entry point. + + Args: + entry_point: Installed entry point. + + Returns: + Source path from ``direct_url.json`` when the entry point distribution + was installed from a local file URL, otherwise ``None``. + """ + distribution = getattr(entry_point, "dist", None) + if distribution is None: + return None + + for distribution_file in distribution.files or (): + if not str(distribution_file).endswith("direct_url.json"): + continue + direct_url_path = distribution.locate_file(distribution_file) + try: + direct_url = json.loads(direct_url_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + + url = direct_url.get("url") + if not isinstance(url, str): + return None + parsed = urlparse(url) + if parsed.scheme != "file": + return None + return Path(unquote(parsed.path)).resolve() + + return None + + +def entry_point_distribution_name(entry_point: importlib.metadata.EntryPoint) -> str | None: + """Return the distribution name that owns an entry point. + + Args: + entry_point: Installed entry point. + + Returns: + Owning distribution name, or ``None`` if it cannot be determined. + """ + distribution = getattr(entry_point, "dist", None) + if distribution is None: + return None + return distribution.metadata.get("Name") + + +def normalize_distribution_name(name: str) -> str: + """Normalize a Python distribution name for comparison. + + Args: + name: Distribution name. + + Returns: + PEP 503-style normalized distribution name. + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def render_catalog_json(entries: list[CatalogEntry]) -> str: + """Render catalog entries as deterministic JSON. + + Args: + entries: Catalog entries to render. + + Returns: + JSON catalog content. + """ + catalog = { + "schema_version": CATALOG_SCHEMA_VERSION, + "plugins": [ + { + "name": entry.name, + "plugin_type": entry.plugin_type, + "description": entry.description, + "package": { + "name": entry.plugin_package, + "version": entry.version, + "path": entry.repository_path, + }, + "entry_point": { + "group": PLUGIN_ENTRY_POINT_GROUP, + "name": entry.entry_point_name, + "value": entry.entry_point_value, + }, + "compatibility": { + "python": { + "specifier": entry.python_requires, + }, + "data_designer": { + "requirement": entry.data_designer_requirement, + "specifier": entry.data_designer_version_specifier, + "marker": entry.data_designer_marker, + }, + }, + } + for entry in entries + ], + } + return f"{json.dumps(catalog, indent=2)}\n" + + +if __name__ == "__main__": + main() diff --git a/devtools/ddp/src/ddp/cli.py b/devtools/ddp/src/ddp/cli.py index cb7861c..112a125 100644 --- a/devtools/ddp/src/ddp/cli.py +++ b/devtools/ddp/src/ddp/cli.py @@ -8,6 +8,7 @@ ddp --help # List all subcommands ddp new my-plugin # Scaffold a new plugin ddp plugin-docs # Generate plugin documentation pages + ddp sync catalog # Sync generated catalog JSON ddp validate # Validate all installed plugins ddp bump patch # Bump a plugin version """ @@ -59,6 +60,32 @@ def build_parser() -> argparse.ArgumentParser: ) p_plugin_docs.set_defaults(func=_run_plugin_docs) + # ddp sync + p_sync = sub.add_parser( + "sync", + help="Sync generated repository artifacts", + description=( + "Sync generated repository artifacts from the current workspace state. " + "Use subcommands such as `ddp sync catalog` for individual artifacts." + ), + ) + sync_sub = p_sync.add_subparsers(dest="sync_target", required=True) + + p_sync_catalog = sync_sub.add_parser( + "catalog", + help="Sync plugin catalog JSON", + description=( + "Sync catalog/plugins.json from installed local DataDesigner plugins and package metadata " + "(package, version, runtime plugin name, type, description, entry point, and compatibility)." + ), + ) + p_sync_catalog.add_argument( + "--check", + action="store_true", + help="Check whether the catalog is current without updating catalog/plugins.json", + ) + p_sync_catalog.set_defaults(func=_run_sync_catalog) + # ddp codeowners p_codeowners = sub.add_parser( "codeowners", @@ -148,6 +175,26 @@ def _run_plugin_docs(args: argparse.Namespace) -> int: return plugin_docs_main(argv) +def _run_sync_catalog(args: argparse.Namespace) -> int: + from ddp.catalog import CatalogError, check_catalog, sync_catalog + + try: + if args.check: + if check_catalog(): + print("Catalog is up to date.") + return 0 + print("ERROR: catalog is out of date; run `uv run ddp sync catalog`.", file=sys.stderr) + return 1 + + output_path = sync_catalog() + except CatalogError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + print(f"Synced catalog: {output_path}") + return 0 + + def _run_codeowners(args: argparse.Namespace) -> int: from ddp.codeowners import main as codeowners_main diff --git a/devtools/ddp/tests/test_catalog.py b/devtools/ddp/tests/test_catalog.py new file mode 100644 index 0000000..eb82d8a --- /dev/null +++ b/devtools/ddp/tests/test_catalog.py @@ -0,0 +1,489 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for ddp.catalog.""" + +from __future__ import annotations + +import io +import json +import textwrap +from contextlib import redirect_stdout +from pathlib import Path + +import pytest + +from ddp import catalog + + +class FakePluginType: + """Plugin type stand-in with the DataDesigner enum ``value`` interface.""" + + def __init__(self, value: str) -> None: + """Initialize the fake plugin type. + + Args: + value: Runtime plugin type value. + """ + self.value = value + + +class FakePlugin: + """Plugin stand-in with the runtime catalog metadata interface.""" + + def __init__(self, name: str, plugin_type: str) -> None: + """Initialize the fake plugin. + + Args: + name: Runtime plugin name. + plugin_type: Runtime plugin type value. + """ + self.name = name + self.plugin_type = FakePluginType(plugin_type) + + +class FakePluginLoader: + """Callable fake entry point loader for catalog row tests.""" + + def __init__(self, plugins: dict[str, FakePlugin]) -> None: + """Initialize the fake loader. + + Args: + plugins: Fake plugins keyed by entry point name. + """ + self.plugins = plugins + self.calls: list[tuple[str, str, str, Path]] = [] + + def __call__( + self, + package_name: str, + entry_point_name: str, + entry_point_value: str, + package_dir: Path, + ) -> FakePlugin: + """Load a fake plugin by entry point name. + + Args: + package_name: Local plugin package name. + entry_point_name: Entry point name in the ``data_designer.plugins`` group. + entry_point_value: Entry point import target. + package_dir: Local plugin package directory. + + Returns: + Fake plugin object. + """ + self.calls.append((package_name, entry_point_name, entry_point_value, package_dir)) + return self.plugins[entry_point_name] + + +class FakeEntryPoint: + """Entry point stand-in for installed metadata validation tests.""" + + def __init__(self, name: str, value: str) -> None: + """Initialize the fake entry point. + + Args: + name: Entry point name. + value: Entry point import target. + """ + self.name = name + self.value = value + + +def write_plugin_pyproject( + plugins_dir: Path, + package_name: str, + version: str, + description: str, + entry_points: dict[str, str] | None, + dependencies: list[str] | None = None, + requires_python: str = ">=3.10", +) -> None: + """Write a minimal plugin pyproject for catalog tests. + + Args: + plugins_dir: Temporary ``plugins/`` directory. + package_name: Package name for ``[project].name``. + version: Package version for ``[project].version``. + description: Package description for ``[project].description``. + entry_points: Entry points for ``data_designer.plugins``. + dependencies: Requirement strings for ``[project].dependencies``. + requires_python: Python compatibility specifier. + """ + plugin_dir = plugins_dir / package_name + plugin_dir.mkdir(parents=True) + dependencies = dependencies or ["data-designer>=0.5.7"] + dependencies_toml = "[" + ", ".join(f'"{dependency}"' for dependency in dependencies) + "]" + entry_point_section = "" + if entry_points is not None: + entry_point_lines = "\n".join(f'{name} = "{value}"' for name, value in entry_points.items()) + entry_point_section = textwrap.dedent( + f""" + + [project.entry-points."data_designer.plugins"] + {entry_point_lines} + """ + ) + pyproject = textwrap.dedent( + f""" + [project] + name = "{package_name}" + version = "{version}" + description = "{description}" + requires-python = "{requires_python}" + dependencies = {dependencies_toml} + {entry_point_section} + """ + ).lstrip() + (plugin_dir / "pyproject.toml").write_text(pyproject, encoding="utf-8") + + +def test_main_produces_json_catalog() -> None: + buf = io.StringIO() + with redirect_stdout(buf): + catalog.main() + output = json.loads(buf.getvalue()) + assert output["schema_version"] == 1 + assert isinstance(output["plugins"], list) + + +def test_discover_catalog_entries_uses_entry_point_runtime_metadata(monkeypatch, tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-multi", + version="1.2.3", + description="Package-level description", + entry_points={ + "z-entry": "example.plugin:z_plugin", + "a-entry": "example.plugin:a_plugin", + }, + ) + loader = FakePluginLoader( + { + "z-entry": FakePlugin(name="z-runtime-name", plugin_type="processor"), + "a-entry": FakePlugin(name="a-runtime-name", plugin_type="seed-reader"), + } + ) + monkeypatch.setattr(catalog, "load_plugin_from_entry_point", loader) + + entries = catalog.discover_catalog_entries(plugins_dir) + + assert loader.calls == [ + ("data-designer-multi", "a-entry", "example.plugin:a_plugin", plugins_dir / "data-designer-multi"), + ("data-designer-multi", "z-entry", "example.plugin:z_plugin", plugins_dir / "data-designer-multi"), + ] + assert entries == [ + catalog.CatalogEntry( + plugin_package="data-designer-multi", + version="1.2.3", + name="a-runtime-name", + plugin_type="seed-reader", + description="Package-level description", + entry_point_name="a-entry", + entry_point_value="example.plugin:a_plugin", + repository_path="plugins/data-designer-multi", + python_requires=">=3.10", + data_designer_requirement="data-designer>=0.5.7", + data_designer_version_specifier=">=0.5.7", + data_designer_marker=None, + ), + catalog.CatalogEntry( + plugin_package="data-designer-multi", + version="1.2.3", + name="z-runtime-name", + plugin_type="processor", + description="Package-level description", + entry_point_name="z-entry", + entry_point_value="example.plugin:z_plugin", + repository_path="plugins/data-designer-multi", + python_requires=">=3.10", + data_designer_requirement="data-designer>=0.5.7", + data_designer_version_specifier=">=0.5.7", + data_designer_marker=None, + ), + ] + + +def test_render_catalog_json_outputs_plugin_compatibility_contract() -> None: + output = catalog.render_catalog_json( + [ + catalog.CatalogEntry( + plugin_package="data-designer-example", + version="0.2.0", + name="runtime-name", + plugin_type="column-generator", + description="Package description", + entry_point_name="runtime-entry", + entry_point_value="example.plugin:plugin", + repository_path="plugins/data-designer-example", + python_requires=">=3.10", + data_designer_requirement='data-designer>=0.5.7,<0.6; python_version >= "3.10"', + data_designer_version_specifier=">=0.5.7,<0.6", + data_designer_marker='python_version >= "3.10"', + ) + ] + ) + data = json.loads(output) + + assert data == { + "schema_version": 1, + "plugins": [ + { + "name": "runtime-name", + "plugin_type": "column-generator", + "description": "Package description", + "package": { + "name": "data-designer-example", + "version": "0.2.0", + "path": "plugins/data-designer-example", + }, + "entry_point": { + "group": "data_designer.plugins", + "name": "runtime-entry", + "value": "example.plugin:plugin", + }, + "compatibility": { + "python": { + "specifier": ">=3.10", + }, + "data_designer": { + "requirement": 'data-designer>=0.5.7,<0.6; python_version >= "3.10"', + "specifier": ">=0.5.7,<0.6", + "marker": 'python_version >= "3.10"', + }, + }, + } + ], + } + + +def test_sync_and_check_catalog_use_default_repo_path(monkeypatch, tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + catalog_path = tmp_path / "catalog" / "plugins.json" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-example", + version="0.2.0", + description="Package description", + entry_points={"runtime-entry": "example.plugin:plugin"}, + ) + loader = FakePluginLoader({"runtime-entry": FakePlugin(name="runtime-name", plugin_type="column-generator")}) + monkeypatch.setattr(catalog, "PLUGINS_DIR", plugins_dir) + monkeypatch.setattr(catalog, "CATALOG_BASE_PATH", tmp_path / "catalog") + monkeypatch.setattr(catalog, "PLUGINS_CATALOG_PATH", catalog_path) + monkeypatch.setattr(catalog, "load_plugin_from_entry_point", loader) + + output_path = catalog.sync_catalog() + + assert output_path == catalog_path + assert catalog.check_catalog() + assert json.loads(output_path.read_text(encoding="utf-8"))["plugins"][0]["name"] == "runtime-name" + + output_path.write_text("{}\n", encoding="utf-8") + assert not catalog.check_catalog() + assert (tmp_path / "catalog" / "plugins.json.new").is_file() + + +def test_missing_installed_entry_point_error_names_package_and_entry_point() -> None: + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.find_installed_entry_point( + package_name="data-designer-ddp-test-missing", + entry_point_name="missing-entry", + entry_point_value="missing.module:plugin", + package_dir=Path("plugins/data-designer-ddp-test-missing"), + ) + + message = str(exc_info.value) + assert "data-designer-ddp-test-missing" in message + assert "missing-entry" in message + + +def test_missing_entry_point_group_errors(tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-missing-entry-points", + version="0.2.0", + description="Package description", + entry_points=None, + ) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.discover_catalog_entries(plugins_dir) + + message = str(exc_info.value) + assert "data-designer-missing-entry-points" in message + assert catalog.PLUGIN_ENTRY_POINT_GROUP in message + + +def test_malformed_entry_point_group_errors() -> None: + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.data_designer_entry_points( + package_name="data-designer-malformed-entry-points", + project={"entry-points": {catalog.PLUGIN_ENTRY_POINT_GROUP: {"runtime-entry": 42}}}, + ) + + message = str(exc_info.value) + assert "data-designer-malformed-entry-points" in message + assert "runtime-entry" in message + assert "expected a non-empty string" in message + + +def test_invalid_project_version_errors(tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-invalid-version", + version="unknown", + description="Package description", + entry_points={"runtime-entry": "example.plugin:plugin"}, + ) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.discover_catalog_entries(plugins_dir) + + message = str(exc_info.value) + assert "data-designer-invalid-version" in message + assert "[project].version" in message + + +def test_invalid_python_requires_errors(tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-invalid-python", + version="0.2.0", + description="Package description", + entry_points={"runtime-entry": "example.plugin:plugin"}, + requires_python="not a specifier", + ) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.discover_catalog_entries(plugins_dir) + + message = str(exc_info.value) + assert "data-designer-invalid-python" in message + assert "requires-python" in message + + +def test_stale_installed_entry_point_target_errors(monkeypatch, tmp_path: Path) -> None: + package_dir = tmp_path / "plugins" / "data-designer-example" + package_dir.mkdir(parents=True) + monkeypatch.setattr(catalog, "entry_point_distribution_source_path", lambda _entry_point: package_dir) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.validate_installed_entry_point( + package_name="data-designer-example", + entry_point=FakeEntryPoint("runtime-entry", "stale.module:plugin"), + entry_point_value="example.plugin:plugin", + package_dir=package_dir, + ) + + message = str(exc_info.value) + assert "data-designer-example" in message + assert "stale" in message + assert "example.plugin:plugin" in message + + +def test_stale_installed_entry_point_source_errors(monkeypatch, tmp_path: Path) -> None: + package_dir = tmp_path / "plugins" / "data-designer-example" + package_dir.mkdir(parents=True) + stale_dir = tmp_path / "stale" / "data-designer-example" + stale_dir.mkdir(parents=True) + monkeypatch.setattr(catalog, "entry_point_distribution_source_path", lambda _entry_point: stale_dir) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.validate_installed_entry_point( + package_name="data-designer-example", + entry_point=FakeEntryPoint("runtime-entry", "example.plugin:plugin"), + entry_point_value="example.plugin:plugin", + package_dir=package_dir, + ) + + message = str(exc_info.value) + assert "data-designer-example" in message + assert stale_dir.as_posix() in message + assert package_dir.as_posix() in message + + +def test_missing_data_designer_dependency_errors(tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-missing-dependency", + version="0.2.0", + description="Package description", + entry_points={"runtime-entry": "example.plugin:plugin"}, + dependencies=["requests>=2"], + ) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.discover_catalog_entries(plugins_dir) + + message = str(exc_info.value) + assert "data-designer-missing-dependency" in message + assert "data-designer" in message + + +def test_malformed_dependencies_errors() -> None: + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.data_designer_requirement_for_dependencies( + package_name="data-designer-malformed-dependency", + dependencies={"data-designer": ">=0.5.7"}, + ) + + message = str(exc_info.value) + assert "data-designer-malformed-dependency" in message + assert "list of strings" in message + + +def test_unversioned_data_designer_dependency_errors(tmp_path: Path) -> None: + plugins_dir = tmp_path / "plugins" + write_plugin_pyproject( + plugins_dir=plugins_dir, + package_name="data-designer-unversioned-dependency", + version="0.2.0", + description="Package description", + entry_points={"runtime-entry": "example.plugin:plugin"}, + dependencies=["data-designer"], + ) + + with pytest.raises(catalog.CatalogError) as exc_info: + catalog.discover_catalog_entries(plugins_dir) + + message = str(exc_info.value) + assert "data-designer-unversioned-dependency" in message + assert "version specifier" in message + + +def test_main_includes_template_plugin() -> None: + buf = io.StringIO() + with redirect_stdout(buf): + catalog.main() + output = json.loads(buf.getvalue()) + assert { + "name": "text-transform", + "plugin_type": "column-generator", + "description": "Template Data Designer plugin — text transform column generator", + "package": { + "name": "data-designer-template", + "version": "0.1.0", + "path": "plugins/data-designer-template", + }, + "entry_point": { + "group": "data_designer.plugins", + "name": "text-transform", + "value": "data_designer_template.plugin:plugin", + }, + "compatibility": { + "python": { + "specifier": ">=3.10", + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": None, + }, + }, + } in output["plugins"] diff --git a/devtools/ddp/tests/test_cli.py b/devtools/ddp/tests/test_cli.py index 2e05d5d..d4cad46 100644 --- a/devtools/ddp/tests/test_cli.py +++ b/devtools/ddp/tests/test_cli.py @@ -18,6 +18,7 @@ class TestBuildParser: EXPECTED_COMMANDS = ( "new", "plugin-docs", + "sync", "codeowners", "license-headers", "validate", @@ -71,6 +72,13 @@ def test_plugin_docs_check_flag(self) -> None: args = parser.parse_args(["plugin-docs", "--check"]) assert args.check is True + def test_sync_catalog_parses_args(self) -> None: + parser = build_parser() + args = parser.parse_args(["sync", "catalog", "--check"]) + assert args.command == "sync" + assert args.sync_target == "catalog" + assert args.check is True + def test_no_command_prints_help(self) -> None: parser = build_parser() args = parser.parse_args([]) @@ -96,6 +104,14 @@ def test_plugin_docs_dispatches(self, mock_run: MagicMock) -> None: args.func(args) mock_run.assert_called_once_with(args) + @patch("ddp.cli._run_sync_catalog") + def test_sync_catalog_dispatches(self, mock_run: MagicMock) -> None: + mock_run.return_value = 0 + parser = build_parser() + args = parser.parse_args(["sync", "catalog"]) + args.func(args) + mock_run.assert_called_once_with(args) + @patch("ddp.cli._run_validate") def test_validate_dispatches(self, mock_run: MagicMock) -> None: mock_run.return_value = 0 diff --git a/docs/authoring.md b/docs/authoring.md index e63f502..90bad30 100644 --- a/docs/authoring.md +++ b/docs/authoring.md @@ -125,8 +125,16 @@ When plugin docs, plugin metadata, or ownership changes, regenerate the derived files: ```bash +make sync make plugin-docs +make catalog make codeowners ``` -CI verifies that generated plugin docs and `.github/CODEOWNERS` are current. +CI verifies that generated plugin docs, `catalog/plugins.json`, and +`.github/CODEOWNERS` are current. The catalog's +`compatibility.data_designer.requirement` and +`compatibility.data_designer.specifier` fields come from each package's direct +versioned `data-designer` dependency in `[project].dependencies`. The catalog +also publishes the package's `requires-python` specifier and any +`data-designer` dependency environment marker. diff --git a/docs/index.md b/docs/index.md index 489dc84..464babf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,4 +38,6 @@ DataDesignerPlugins/ - Write tests around public interfaces. - Regenerate generated metadata when plugin docs, plugin metadata, or ownership changes. +- Keep `catalog/plugins.json` current when plugin package metadata or entry + points change. - Run the Makefile targets locally before opening or updating a pull request. diff --git a/docs/workflow.md b/docs/workflow.md index 2f2fcf2..72c7f1b 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -27,7 +27,7 @@ The target runs: | `make lint` | Ruff linting and formatting. | | `make test` | Each plugin's tests in an isolated virtual environment. | | `make validate` | Installed `data_designer.plugins` entry points with `assert_valid_plugin`. | -| `make check` | Generated plugin docs, generated CODEOWNERS, and SPDX license headers. | +| `make check` | Generated plugin docs, generated catalog, generated CODEOWNERS, and SPDX license headers. | | `make docs` | Zensical builds the documentation site in strict mode. | | `make docs-server` | Zensical serves the documentation site locally while you edit. | @@ -64,12 +64,15 @@ Generated site inputs come from repository metadata and plugin docs: ```bash make plugin-docs +make catalog make codeowners ``` `docs/plugins/` and the plugin section of `zensical.toml` are generated from plugin package metadata and `plugins/*/docs/`. Do not edit generated plugin site -pages directly. +pages directly. `catalog/plugins.json` is generated from installed local plugin +entry points, package metadata, and direct `data-designer` dependency specifiers +for compatibility checks by external tools. ## GitHub CI diff --git a/uv.lock b/uv.lock index 97c44fd..781f8c8 100644 --- a/uv.lock +++ b/uv.lock @@ -453,11 +453,15 @@ name = "ddp" version = "0.1.0" source = { editable = "devtools/ddp" } dependencies = [ + { name = "packaging" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] [package.metadata] -requires-dist = [{ name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=1.1.0" }] +requires-dist = [ + { name = "packaging", specifier = ">=24.0" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=1.1.0" }, +] [[package]] name = "deepmerge"