From fce1016b0d54480e3a8ab62828a37ae926a6d33e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 20:28:54 +0000 Subject: [PATCH 01/34] feat(cli): add plugin catalog services Add typed catalog and tap models, persistent tap storage, cached catalog loading, compatibility evaluation, install plan generation, and runtime plugin discovery helpers. Refs #617 --- .../src/data_designer/cli/plugin_catalog.py | 156 +++++++++ .../cli/repositories/plugin_tap_repository.py | 297 ++++++++++++++++++ .../cli/services/plugin_catalog_service.py | 279 ++++++++++++++++ .../cli/services/plugin_install_service.py | 158 ++++++++++ uv.lock | 6 +- 5 files changed, 893 insertions(+), 3 deletions(-) create mode 100644 packages/data-designer/src/data_designer/cli/plugin_catalog.py create mode 100644 packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py create mode 100644 packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py create mode 100644 packages/data-designer/src/data_designer/cli/services/plugin_install_service.py diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py new file mode 100644 index 000000000..4cbebfe37 --- /dev/null +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field + +from data_designer.plugins.plugin import PluginType + +DEFAULT_PLUGIN_TAP_ALIAS = "nvidia" +DEFAULT_PLUGIN_TAP_URL = "https://raw.githubusercontent.com/NVIDIA-NeMo/DataDesignerPlugins/main/catalog/plugins.json" +PLUGIN_TAPS_FILE_NAME = "plugin_taps.yaml" +PLUGIN_TAP_CACHE_DIR_NAME = "plugin-tap-cache" +PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS = 24 * 60 * 60 +MAX_PLUGIN_CATALOG_SIZE_BYTES = 1 * 1024 * 1024 +SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS = {1, 2} +PLUGIN_TAP_ALIAS_PATTERN = r"^[A-Za-z0-9_.-]+$" + + +class PluginCatalogError(ValueError): + """Raised when a plugin catalog cannot be loaded or validated.""" + + +class PluginCompatibilityTarget(BaseModel): + """Version requirement for one environment target.""" + + model_config = ConfigDict(extra="allow") + + specifier: str | None = None + marker: str | None = None + + +class PluginCompatibility(BaseModel): + """Compatibility requirements declared by a catalog entry.""" + + model_config = ConfigDict(extra="allow") + + python: PluginCompatibilityTarget | None = None + data_designer: PluginCompatibilityTarget | None = None + + +class PluginPackageInfo(BaseModel): + """Python package metadata for a catalog entry.""" + + model_config = ConfigDict(extra="allow") + + name: str + version: str | None = None + path: str | None = None + + +class PluginEntryPointInfo(BaseModel): + """Runtime entry point exposed by an installable plugin package.""" + + model_config = ConfigDict(extra="allow") + + group: str = "data_designer.plugins" + name: str + value: str + + +class PluginSourceInfo(BaseModel): + """Install source metadata for a catalog entry.""" + + model_config = ConfigDict(extra="allow") + + type: str + package: str | None = None + url: str | None = None + ref: str | None = None + path: str | None = None + subdirectory: str | None = None + editable: bool = False + + +class PluginDocsInfo(BaseModel): + """Documentation metadata for a catalog entry.""" + + model_config = ConfigDict(extra="allow") + + url: str | None = None + + +class PluginCatalogEntry(BaseModel): + """One discoverable Data Designer plugin entry from a tap catalog.""" + + model_config = ConfigDict(extra="allow") + + name: str + plugin_type: PluginType + description: str = "" + package: PluginPackageInfo + entry_point: PluginEntryPointInfo + compatibility: PluginCompatibility | None = None + source: PluginSourceInfo | None = None + docs: PluginDocsInfo | None = None + tags: list[str] = Field(default_factory=list) + maintainers: list[str] = Field(default_factory=list) + release_notes_url: str | None = None + + +class PluginCatalog(BaseModel): + """Versioned plugin tap catalog.""" + + model_config = ConfigDict(extra="allow") + + schema_version: int + plugins: list[PluginCatalogEntry] = Field(default_factory=list) + + +class PluginTapConfig(BaseModel): + """Persisted tap configuration.""" + + alias: str = Field(pattern=PLUGIN_TAP_ALIAS_PATTERN) + url: str + trusted: bool = False + cache_ttl_seconds: int = Field(default=PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, ge=0) + + +class PluginTapRegistry(BaseModel): + """Persisted collection of user-configured plugin taps.""" + + taps: list[PluginTapConfig] = Field(default_factory=list) + + +@dataclass(frozen=True) +class CompatibilityResult: + """Compatibility result for one catalog entry in the local environment.""" + + is_compatible: bool + reasons: list[str] + + +@dataclass(frozen=True) +class InstallPlan: + """Resolved package-manager command for installing one plugin entry.""" + + plugin_name: str + package_name: str + source_description: str + command: list[str] + manager: str + tap_alias: str + trusted_tap: bool + + +@dataclass(frozen=True) +class InstalledPluginInfo: + """Runtime plugin discovered from installed entry points.""" + + name: str + plugin_type: PluginType + config_qualified_name: str + impl_qualified_name: str diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py new file mode 100644 index 000000000..5ef0dcbeb --- /dev/null +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from pydantic import ValidationError + +from data_designer.cli.plugin_catalog import ( + DEFAULT_PLUGIN_TAP_ALIAS, + DEFAULT_PLUGIN_TAP_URL, + MAX_PLUGIN_CATALOG_SIZE_BYTES, + PLUGIN_TAP_CACHE_DIR_NAME, + PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, + PLUGIN_TAPS_FILE_NAME, + SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS, + PluginCatalog, + PluginCatalogError, + PluginTapConfig, + PluginTapRegistry, +) +from data_designer.cli.repositories.base import ConfigRepository +from data_designer.config.utils.io_helpers import load_config_file, save_config_file + + +class PluginTapRepository(ConfigRepository[PluginTapRegistry]): + """Repository for plugin tap aliases and cached catalog payloads.""" + + @property + def config_file(self) -> Path: + """Get the plugin tap configuration file path.""" + return self.config_dir / PLUGIN_TAPS_FILE_NAME + + @property + def cache_dir(self) -> Path: + """Get the plugin tap cache directory path.""" + return self.config_dir / PLUGIN_TAP_CACHE_DIR_NAME + + def load(self) -> PluginTapRegistry | None: + """Load user-configured plugin taps.""" + if not self.exists(): + return None + + try: + config_dict = load_config_file(self.config_file) + return PluginTapRegistry.model_validate(config_dict) + except Exception: + return None + + def save(self, config: PluginTapRegistry) -> None: + """Save user-configured plugin taps.""" + config_dict = config.model_dump(mode="json", exclude_none=True) + save_config_file(self.config_file, config_dict) + + def list_taps(self) -> list[PluginTapConfig]: + """Return the built-in NVIDIA tap followed by user-configured taps.""" + taps = [self.default_tap()] + registry = self.load() + if registry is not None: + taps.extend(sorted(registry.taps, key=lambda tap: tap.alias)) + return taps + + def get_tap(self, alias: str | None = None) -> PluginTapConfig | None: + """Return a tap by alias, defaulting to the built-in NVIDIA tap.""" + resolved_alias = alias or DEFAULT_PLUGIN_TAP_ALIAS + return next((tap for tap in self.list_taps() if tap.alias == resolved_alias), None) + + def add_tap( + self, + alias: str, + url: str, + *, + trusted: bool = False, + cache_ttl_seconds: int = PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, + ) -> PluginTapConfig: + """Persist a new tap alias. + + Raises: + ValueError: If the alias already exists or is reserved for the built-in tap. + """ + if self.get_tap(alias) is not None: + raise ValueError(f"Plugin tap alias {alias!r} already exists") + + tap = PluginTapConfig( + alias=alias, + url=normalize_tap_location(url), + trusted=trusted, + cache_ttl_seconds=cache_ttl_seconds, + ) + registry = self.load() or PluginTapRegistry() + registry.taps.append(tap) + registry.taps = sorted(registry.taps, key=lambda item: item.alias) + self.save(registry) + return tap + + def remove_tap(self, alias: str) -> None: + """Remove a user-configured tap alias. + + Raises: + ValueError: If the alias is reserved or does not exist. + """ + if alias == DEFAULT_PLUGIN_TAP_ALIAS: + raise ValueError(f"Cannot remove the built-in {DEFAULT_PLUGIN_TAP_ALIAS!r} plugin tap") + + registry = self.load() + if registry is None or not any(tap.alias == alias for tap in registry.taps): + raise ValueError(f"Plugin tap alias {alias!r} not found") + + registry.taps = [tap for tap in registry.taps if tap.alias != alias] + if registry.taps: + self.save(registry) + else: + self.delete() + + cache_file = self._cache_file(alias) + cache_file.unlink(missing_ok=True) + + def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> PluginCatalog: + """Load a tap catalog from cache or source.""" + tap = self.get_tap(alias) + if tap is None: + raise ValueError(f"Plugin tap alias {alias!r} not found") + + if not refresh: + cached_catalog = self._load_cached_catalog(tap, require_fresh=True) + if cached_catalog is not None: + return cached_catalog + + try: + payload = self._fetch_catalog_payload(tap.url) + catalog = self._validate_catalog(payload, source=tap.url) + except Exception: + if not refresh: + cached_catalog = self._load_cached_catalog(tap, require_fresh=False) + if cached_catalog is not None: + return cached_catalog + raise + + self._save_catalog_cache(tap, payload) + return catalog + + def _load_cached_catalog(self, tap: PluginTapConfig, *, require_fresh: bool) -> PluginCatalog | None: + cache_file = self._cache_file(tap.alias) + if not cache_file.exists(): + return None + + try: + with open(cache_file) as f: + cache_payload = json.load(f) + fetched_at = datetime.fromisoformat(cache_payload["fetched_at"]) + if fetched_at.tzinfo is None: + fetched_at = fetched_at.replace(tzinfo=timezone.utc) + if require_fresh and tap.cache_ttl_seconds == 0: + return None + if require_fresh: + age_seconds = (datetime.now(timezone.utc) - fetched_at).total_seconds() + if age_seconds > tap.cache_ttl_seconds: + return None + catalog_payload = cache_payload["catalog"] + return self._validate_catalog(catalog_payload, source=str(cache_file)) + except Exception: + return None + + def _save_catalog_cache(self, tap: PluginTapConfig, catalog_payload: dict) -> None: + self.cache_dir.mkdir(parents=True, exist_ok=True) + cache_payload = { + "tap_alias": tap.alias, + "tap_url": tap.url, + "fetched_at": datetime.now(timezone.utc).isoformat(), + "catalog": catalog_payload, + } + with open(self._cache_file(tap.alias), "w") as f: + json.dump(cache_payload, f, indent=2, sort_keys=True) + + def _cache_file(self, alias: str) -> Path: + return self.cache_dir / f"{alias}.json" + + @staticmethod + def _fetch_catalog_payload(location: str) -> dict: + if _is_http_url(location): + return _fetch_remote_catalog(location) + return _fetch_local_catalog(location) + + @staticmethod + def _validate_catalog(payload: dict, *, source: str) -> PluginCatalog: + try: + catalog = PluginCatalog.model_validate(payload) + except ValidationError as e: + raise PluginCatalogError(f"Invalid plugin catalog at {source!r}: {e}") from e + + if catalog.schema_version not in SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS: + supported = ", ".join(str(version) for version in sorted(SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS)) + raise PluginCatalogError( + f"Unsupported plugin catalog schema_version {catalog.schema_version!r} at {source!r}. " + f"Supported versions: {supported}." + ) + return catalog + + @staticmethod + def default_tap() -> PluginTapConfig: + """Return the built-in NVIDIA plugin tap configuration.""" + return PluginTapConfig( + alias=DEFAULT_PLUGIN_TAP_ALIAS, + url=DEFAULT_PLUGIN_TAP_URL, + trusted=True, + cache_ttl_seconds=PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, + ) + + +def normalize_tap_location(location: str) -> str: + """Normalize a tap repository, catalog URL, or local path to a catalog location.""" + if _is_http_url(location): + return _normalize_tap_url(location) + + path = Path(location).expanduser() + if path.suffix.lower() == ".json": + return str(path.resolve(strict=False)) + return str((path / "catalog" / "plugins.json").resolve(strict=False)) + + +def _normalize_tap_url(url: str) -> str: + parsed = urlparse(url) + hostname = parsed.hostname or "" + segments = [segment for segment in parsed.path.split("/") if segment] + + if hostname in {"github.com", "www.github.com"} and len(segments) >= 2: + owner, repo = segments[0], segments[1] + if len(segments) == 2: + return f"https://raw.githubusercontent.com/{owner}/{repo}/main/catalog/plugins.json" + if len(segments) >= 5 and segments[2] == "blob": + ref = segments[3] + path = "/".join(segments[4:]) + return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}" + if len(segments) >= 4 and segments[2] == "tree": + ref = segments[3] + return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/catalog/plugins.json" + + return url + + +def _fetch_local_catalog(location: str) -> dict: + path = Path(location).expanduser() + if not path.exists(): + raise PluginCatalogError(f"Plugin catalog file not found: {path}") + if path.stat().st_size > MAX_PLUGIN_CATALOG_SIZE_BYTES: + raise PluginCatalogError( + f"Plugin catalog at {path} exceeds maximum size of {MAX_PLUGIN_CATALOG_SIZE_BYTES} bytes" + ) + + try: + with open(path) as f: + payload = json.load(f) + except json.JSONDecodeError as e: + raise PluginCatalogError(f"Failed to parse plugin catalog JSON at {path}: {e}") from e + + if not isinstance(payload, dict): + raise PluginCatalogError(f"Plugin catalog at {path} must be a JSON object") + return payload + + +def _fetch_remote_catalog(url: str) -> dict: + request = Request(url, headers={"User-Agent": "data-designer"}) + try: + with urlopen(request, timeout=10) as response: + status = getattr(response, "status", 200) + if isinstance(status, int) and status >= 400: + raise PluginCatalogError(f"Failed to fetch plugin catalog {url!r}: HTTP {status}") + content = response.read(MAX_PLUGIN_CATALOG_SIZE_BYTES + 1) + except HTTPError as e: + raise PluginCatalogError(f"Failed to fetch plugin catalog {url!r}: HTTP {e.code}") from e + except URLError as e: + raise PluginCatalogError(f"Failed to fetch plugin catalog {url!r}: {e.reason}") from e + + if len(content) > MAX_PLUGIN_CATALOG_SIZE_BYTES: + raise PluginCatalogError( + f"Plugin catalog at {url!r} exceeds maximum size of {MAX_PLUGIN_CATALOG_SIZE_BYTES} bytes" + ) + + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as e: + raise PluginCatalogError(f"Failed to parse plugin catalog JSON at {url!r}: {e}") from e + + if not isinstance(payload, dict): + raise PluginCatalogError(f"Plugin catalog at {url!r} must be a JSON object") + return payload + + +def _is_http_url(value: str) -> bool: + parsed = urlparse(value) + return parsed.scheme in {"http", "https"} and bool(parsed.netloc) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py new file mode 100644 index 000000000..7530365cd --- /dev/null +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.metadata +import platform +from collections import defaultdict +from collections.abc import Iterable + +from packaging.markers import InvalidMarker, Marker +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version + +from data_designer.cli.plugin_catalog import ( + DEFAULT_PLUGIN_TAP_ALIAS, + CompatibilityResult, + InstalledPluginInfo, + PluginCatalogEntry, + PluginCompatibilityTarget, + PluginTapConfig, +) +from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository +from data_designer.plugins.plugin import PluginType +from data_designer.plugins.registry import PluginRegistry + + +class PluginCatalogService: + """Business logic for plugin catalog discovery and compatibility checks.""" + + def __init__( + self, + repository: PluginTapRepository, + *, + python_version: str | None = None, + data_designer_version: str | None = None, + ) -> None: + self.repository = repository + self.python_version = python_version or platform.python_version() + self.data_designer_version = data_designer_version or _get_installed_data_designer_version() + + def list_entries( + self, + tap_alias: str | None = None, + *, + refresh: bool = False, + include_incompatible: bool = False, + ) -> list[PluginCatalogEntry]: + """List catalog entries for a tap, filtering incompatible entries by default.""" + catalog = self.repository.load_catalog(tap_alias, refresh=refresh) + entries = sorted(catalog.plugins, key=lambda entry: (entry.name, entry.package.version or "")) + if include_incompatible: + return entries + return [entry for entry in entries if self.evaluate_compatibility(entry).is_compatible] + + def search_entries( + self, + query: str, + tap_alias: str | None = None, + *, + refresh: bool = False, + include_incompatible: bool = False, + ) -> list[PluginCatalogEntry]: + """Search catalog entries by simple token matching.""" + query_tokens = _tokenize(query) + if not query_tokens: + return [] + + return [ + entry + for entry in self.list_entries( + tap_alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + if all(token in _entry_search_text(entry) for token in query_tokens) + ] + + def get_entry( + self, + name: str, + tap_alias: str | None = None, + *, + refresh: bool = False, + include_incompatible: bool = True, + ) -> PluginCatalogEntry: + """Return the newest catalog entry by plugin name.""" + entries = self.list_entries(tap_alias, refresh=refresh, include_incompatible=True) + matches = [entry for entry in entries if entry.name == name] + matched_incompatible = False + if matches and not include_incompatible: + compatible_matches = [entry for entry in matches if self.evaluate_compatibility(entry).is_compatible] + matched_incompatible = bool(matches) and not compatible_matches + matches = compatible_matches + if matches: + return max(matches, key=_entry_version_sort_key) + + resolved_alias = tap_alias or DEFAULT_PLUGIN_TAP_ALIAS + if matched_incompatible: + raise ValueError( + f"Plugin {name!r} was found in tap {resolved_alias!r}, but no compatible version is available" + ) + raise ValueError(f"Plugin {name!r} was not found in tap {resolved_alias!r}") + + @staticmethod + def group_entries_by_package(entries: Iterable[PluginCatalogEntry]) -> dict[str, list[PluginCatalogEntry]]: + """Group catalog entries by installable package name.""" + grouped_entries: dict[str, list[PluginCatalogEntry]] = defaultdict(list) + for entry in entries: + grouped_entries[entry.package.name].append(entry) + return { + package_name: sorted(items, key=lambda item: item.name) for package_name, items in grouped_entries.items() + } + + def evaluate_compatibility(self, entry: PluginCatalogEntry) -> CompatibilityResult: + """Evaluate whether a catalog entry is compatible with the local environment.""" + compatibility = entry.compatibility + if compatibility is None: + return CompatibilityResult(is_compatible=True, reasons=[]) + + reasons = [] + reasons.extend( + self._evaluate_target( + target=compatibility.python, + label="Python", + version=self.python_version, + marker_environment={"python_version": _major_minor(self.python_version)}, + ) + ) + reasons.extend( + self._evaluate_target( + target=compatibility.data_designer, + label="Data Designer", + version=self.data_designer_version, + marker_environment={"python_version": _major_minor(self.python_version)}, + ) + ) + return CompatibilityResult(is_compatible=not reasons, reasons=reasons) + + def list_taps(self) -> list[PluginTapConfig]: + """List available plugin taps.""" + return self.repository.list_taps() + + def get_tap(self, alias: str | None = None) -> PluginTapConfig: + """Return a plugin tap or raise a user-facing error.""" + tap = self.repository.get_tap(alias) + if tap is None: + raise ValueError(f"Plugin tap alias {alias!r} not found") + return tap + + def add_tap( + self, + alias: str, + url: str, + *, + trusted: bool, + cache_ttl_seconds: int, + ) -> PluginTapConfig: + """Add a plugin tap alias.""" + return self.repository.add_tap( + alias, + url, + trusted=trusted, + cache_ttl_seconds=cache_ttl_seconds, + ) + + def remove_tap(self, alias: str) -> None: + """Remove a plugin tap alias.""" + self.repository.remove_tap(alias) + + def list_installed_plugins(self) -> list[InstalledPluginInfo]: + """List runtime plugins currently discoverable through entry points.""" + registry = PluginRegistry() + installed_plugins = [] + for plugin_type in PluginType: + for plugin in registry.get_plugins(plugin_type): + installed_plugins.append( + InstalledPluginInfo( + name=plugin.name, + plugin_type=plugin.plugin_type, + config_qualified_name=plugin.config_qualified_name, + impl_qualified_name=plugin.impl_qualified_name, + ) + ) + return sorted(installed_plugins, key=lambda plugin: (plugin.plugin_type.value, plugin.name)) + + def _evaluate_target( + self, + *, + target: PluginCompatibilityTarget | None, + label: str, + version: str | None, + marker_environment: dict[str, str], + ) -> list[str]: + if target is None or not target.specifier: + return [] + + marker_error = _marker_error(target.marker, marker_environment) + if marker_error is not None: + return [f"{label} marker {target.marker!r} is invalid: {marker_error}"] + if target.marker and not Marker(target.marker).evaluate(marker_environment): + return [] + + if version is None: + return [f"Unable to resolve installed {label} version for constraint {target.specifier!r}"] + + try: + specifier = SpecifierSet(target.specifier) + except InvalidSpecifier as e: + return [f"{label} specifier {target.specifier!r} is invalid: {e}"] + + try: + parsed_version = Version(version) + except InvalidVersion as e: + return [f"Installed {label} version {version!r} is invalid: {e}"] + + if not specifier.contains(parsed_version, prereleases=True): + return [f"{label} {version} does not satisfy {target.specifier}"] + return [] + + +def _get_installed_data_designer_version() -> str | None: + try: + return importlib.metadata.version("data-designer") + except importlib.metadata.PackageNotFoundError: + return None + + +def _tokenize(value: str) -> list[str]: + return [token.strip().lower() for token in value.split() if token.strip()] + + +def _entry_search_text(entry: PluginCatalogEntry) -> str: + values = [ + entry.name, + entry.plugin_type.value, + entry.description, + entry.package.name, + entry.package.version or "", + entry.package.path or "", + entry.entry_point.name, + entry.entry_point.value, + entry.source.type if entry.source is not None else "", + entry.source.package if entry.source is not None and entry.source.package else "", + entry.source.url if entry.source is not None and entry.source.url else "", + entry.docs.url if entry.docs is not None and entry.docs.url else "", + entry.release_notes_url or "", + *_stringify_extra_values(entry.tags), + *_stringify_extra_values(entry.maintainers), + ] + return " ".join(values).lower() + + +def _entry_version_sort_key(entry: PluginCatalogEntry) -> Version: + try: + return Version(entry.package.version or "0") + except InvalidVersion: + return Version("0") + + +def _stringify_extra_values(values: Iterable[str]) -> list[str]: + return [str(value) for value in values] + + +def _major_minor(version: str) -> str: + parts = version.split(".") + if len(parts) < 2: + return version + return ".".join(parts[:2]) + + +def _marker_error(marker: str | None, environment: dict[str, str]) -> str | None: + if marker is None: + return None + try: + Marker(marker).evaluate(environment) + except InvalidMarker as e: + return str(e) + return None diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py new file mode 100644 index 000000000..6f3741c8f --- /dev/null +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import shutil +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from urllib.parse import urlparse + +from data_designer.cli.plugin_catalog import InstallPlan, PluginCatalogEntry, PluginSourceInfo, PluginTapConfig +from data_designer.plugins.registry import PluginRegistry + +InstallRunner = Callable[[list[str]], int] + + +class PluginInstallService: + """Resolve, execute, and verify plugin installation plans.""" + + def __init__(self, runner: InstallRunner | None = None) -> None: + self._runner = runner or _run_subprocess + + def build_install_plan( + self, + entry: PluginCatalogEntry, + tap: PluginTapConfig, + *, + manager: str = "auto", + ) -> InstallPlan: + """Build the exact package-manager command for one catalog entry.""" + resolved_manager = _resolve_manager(manager) + install_args, source_description = _install_args_for_entry(entry, tap) + command = _base_command(resolved_manager) + install_args + return InstallPlan( + plugin_name=entry.name, + package_name=entry.package.name, + source_description=source_description, + command=command, + manager=resolved_manager, + tap_alias=tap.alias, + trusted_tap=tap.trusted, + ) + + def install(self, plan: InstallPlan) -> None: + """Run an installation plan. + + Raises: + RuntimeError: If the package manager exits unsuccessfully. + """ + return_code = self._runner(plan.command) + if return_code != 0: + raise RuntimeError(f"Plugin installer exited with status {return_code}") + + def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: + """Verify the plugin is discoverable by the runtime PluginRegistry.""" + PluginRegistry.reset() + registry = PluginRegistry() + return registry.plugin_exists(entry.name) + + +def _run_subprocess(command: list[str]) -> int: + result = subprocess.run(command, check=False) + return result.returncode + + +def _resolve_manager(manager: str) -> str: + if manager not in {"auto", "uv", "pip"}: + raise ValueError(f"Unsupported plugin installer {manager!r}. Expected 'auto', 'uv', or 'pip'.") + if manager == "auto": + return "uv" if shutil.which("uv") else "pip" + if manager == "uv" and not shutil.which("uv"): + raise ValueError("uv was requested for plugin installation, but it is not available on PATH") + return manager + + +def _base_command(manager: str) -> list[str]: + if manager == "uv": + return ["uv", "pip", "install", "--python", sys.executable] + return [sys.executable, "-m", "pip", "install"] + + +def _install_args_for_entry(entry: PluginCatalogEntry, tap: PluginTapConfig) -> tuple[list[str], str]: + source = entry.source + if source is None: + raise ValueError( + f"Plugin {entry.name!r} cannot be installed because the catalog entry does not declare a source" + ) + + source_type = source.type.lower() + if source_type == "pypi": + target = _pypi_target(entry, source) + return [target], target + if source_type == "git": + target = _git_target(source) + return [target], target + if source_type == "path": + args = _path_args(entry, source, tap) + return args, " ".join(args) + if source_type == "url": + target = _required(source.url, "url", source_type) + return [target], target + + raise ValueError(f"Plugin {entry.name!r} declares unsupported install source type {source.type!r}") + + +def _pypi_target(entry: PluginCatalogEntry, source: PluginSourceInfo) -> str: + package_name = source.package or entry.package.name + if entry.package.version: + return f"{package_name}=={entry.package.version}" + return package_name + + +def _git_target(source: PluginSourceInfo) -> str: + url = _required(source.url, "url", "git") + target = url if url.startswith("git+") else f"git+{url}" + if source.ref: + target = f"{target}@{source.ref}" + + fragments = [] + if source.subdirectory: + fragments.append(f"subdirectory={source.subdirectory}") + if fragments: + target = f"{target}#{'&'.join(fragments)}" + return target + + +def _path_args(entry: PluginCatalogEntry, source: PluginSourceInfo, tap: PluginTapConfig) -> list[str]: + path = source.path or entry.package.path + if path is None: + raise ValueError(f"Plugin {entry.name!r} declares a path source without a path") + + normalized_path = str(_resolve_path_source(path, tap)) + if source.editable: + return ["-e", normalized_path] + return [normalized_path] + + +def _required(value: str | None, field_name: str, source_type: str) -> str: + if value is None: + raise ValueError(f"Plugin install source type {source_type!r} requires {field_name!r}") + return value + + +def _resolve_path_source(path: str, tap: PluginTapConfig) -> Path: + source_path = Path(path).expanduser() + if source_path.is_absolute(): + return source_path + + tap_location = urlparse(tap.url) + if tap_location.scheme in {"http", "https"}: + raise ValueError("Relative path plugin sources require a local plugin tap") + + tap_catalog_path = Path(tap.url).expanduser() + if tap_catalog_path.name == "plugins.json" and tap_catalog_path.parent.name == "catalog": + return tap_catalog_path.parent.parent / source_path + return tap_catalog_path.parent / source_path diff --git a/uv.lock b/uv.lock index 1ba121034..2736484ed 100644 --- a/uv.lock +++ b/uv.lock @@ -3014,11 +3014,11 @@ wheels = [ [[package]] name = "packaging" -version = "26.0" +version = "25.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] [[package]] From bb9428e801eb6082e843a5bb12ee345e6209d2f7 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 20:29:03 +0000 Subject: [PATCH 02/34] feat(cli): add plugins command group Wire list, search, info, install, installed, and tap management commands through the existing command-controller CLI pattern. Refs #617 --- .../src/data_designer/cli/commands/plugins.py | 196 ++++++++++ .../controllers/plugin_catalog_controller.py | 334 ++++++++++++++++++ .../src/data_designer/cli/main.py | 75 ++++ 3 files changed, 605 insertions(+) create mode 100644 packages/data-designer/src/data_designer/cli/commands/plugins.py create mode 100644 packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugins.py new file mode 100644 index 000000000..de2b1afff --- /dev/null +++ b/packages/data-designer/src/data_designer/cli/commands/plugins.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import click +import typer + +from data_designer.cli.controllers.plugin_catalog_controller import PluginCatalogController +from data_designer.config.utils.constants import DATA_DESIGNER_HOME + + +def list_command( + ctx: typer.Context, + tap: str | None = typer.Option( + None, + "--tap", + help="Plugin tap alias to read. Can also be provided before the subcommand.", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Fetch the tap catalog even when a fresh cache entry exists.", + ), + include_incompatible: bool = typer.Option( + False, + "--include-incompatible", + help="Show plugins that do not satisfy the local Python or Data Designer version.", + ), +) -> None: + """List discoverable Data Designer plugins from a tap catalog.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_list( + tap_alias=_resolve_tap_alias(ctx, tap), + refresh=refresh, + include_incompatible=include_incompatible, + ) + + +def search_command( + ctx: typer.Context, + query: str = typer.Argument(help="Keyword, plugin type, package name, source, maintainer, or tag to search for."), + tap: str | None = typer.Option( + None, + "--tap", + help="Plugin tap alias to search. Can also be provided before the subcommand.", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Fetch the tap catalog even when a fresh cache entry exists.", + ), + include_incompatible: bool = typer.Option( + False, + "--include-incompatible", + help="Search plugins that do not satisfy the local Python or Data Designer version.", + ), +) -> None: + """Search discoverable Data Designer plugins from a tap catalog.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_search( + query, + tap_alias=_resolve_tap_alias(ctx, tap), + refresh=refresh, + include_incompatible=include_incompatible, + ) + + +def info_command( + ctx: typer.Context, + plugin_name: str = typer.Argument(help="Plugin name from the tap catalog."), + tap: str | None = typer.Option( + None, + "--tap", + help="Plugin tap alias to read. Can also be provided before the subcommand.", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Fetch the tap catalog even when a fresh cache entry exists.", + ), +) -> None: + """Show metadata, compatibility, docs, and install plan for one plugin.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_info( + plugin_name, + tap_alias=_resolve_tap_alias(ctx, tap), + refresh=refresh, + ) + + +def install_command( + ctx: typer.Context, + plugin_name: str = typer.Argument(help="Plugin name from the tap catalog."), + tap: str | None = typer.Option( + None, + "--tap", + help="Plugin tap alias to install from. Can also be provided before the subcommand.", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Fetch the tap catalog even when a fresh cache entry exists.", + ), + manager: str = typer.Option( + "auto", + "--manager", + click_type=click.Choice(["auto", "uv", "pip"]), + help="Package manager to use for installation.", + ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Install without an interactive confirmation prompt.", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Print the install plan without mutating the current environment.", + ), + force: bool = typer.Option( + False, + "--force", + help="Allow installation even when catalog compatibility checks fail.", + ), +) -> None: + """Install one Data Designer plugin package, then verify runtime discovery.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_install( + plugin_name, + tap_alias=_resolve_tap_alias(ctx, tap), + refresh=refresh, + manager=manager, + yes=yes, + dry_run=dry_run, + force=force, + ) + + +def installed_command() -> None: + """List installed Data Designer plugins discovered from runtime entry points.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_installed() + + +def taps_list_command() -> None: + """List configured plugin taps.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_taps_list() + + +def taps_add_command( + alias: str = typer.Argument(help="Local alias for the plugin tap."), + url: str = typer.Argument(help="Tap repository URL, catalog URL, local catalog file, or local tap directory."), + trusted: bool = typer.Option( + False, + "--trusted", + help="Mark the tap as trusted for install-plan display and confirmations.", + ), + cache_ttl_seconds: int = typer.Option( + 24 * 60 * 60, + "--cache-ttl-seconds", + min=0, + help="Seconds before cached catalog metadata is refreshed. Use 0 to always refresh.", + ), +) -> None: + """Add a plugin tap alias.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_taps_add( + alias=alias, + url=url, + trusted=trusted, + cache_ttl_seconds=cache_ttl_seconds, + ) + + +def taps_remove_command( + alias: str = typer.Argument(help="Plugin tap alias to remove."), +) -> None: + """Remove a plugin tap alias.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_taps_remove(alias=alias) + + +def _resolve_tap_alias(ctx: typer.Context, tap_alias: str | None) -> str | None: + if tap_alias is not None: + return tap_alias + + parent = ctx.parent + while parent is not None: + candidate = parent.params.get("tap") if parent.params else None + if isinstance(candidate, str) and candidate: + return candidate + parent = parent.parent + return None diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py new file mode 100644 index 000000000..2bafd6102 --- /dev/null +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import shlex +from pathlib import Path + +import typer +from rich.table import Table + +from data_designer.cli.plugin_catalog import ( + DEFAULT_PLUGIN_TAP_ALIAS, + CompatibilityResult, + InstalledPluginInfo, + PluginCatalogEntry, + PluginTapConfig, +) +from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository +from data_designer.cli.services.plugin_catalog_service import PluginCatalogService +from data_designer.cli.services.plugin_install_service import PluginInstallService +from data_designer.cli.ui import ( + confirm_action, + console, + display_config_preview, + print_error, + print_header, + print_info, + print_success, + print_warning, +) +from data_designer.config.utils.constants import NordColor + + +class PluginCatalogController: + """Controller for plugin catalog, tap, and install workflows.""" + + def __init__(self, config_dir: Path) -> None: + self.config_dir = config_dir + self.tap_repository = PluginTapRepository(config_dir) + self.catalog_service = PluginCatalogService(self.tap_repository) + self.install_service = PluginInstallService() + + def run_list( + self, + *, + tap_alias: str | None = None, + refresh: bool = False, + include_incompatible: bool = False, + ) -> None: + """List plugins from a tap catalog.""" + tap = self._get_tap_or_exit(tap_alias) + entries = self._list_entries_or_exit(tap.alias, refresh=refresh, include_incompatible=include_incompatible) + + print_header("Data Designer Plugins") + print_info(f"Tap: {tap.alias} ({tap.url})") + console.print() + + if not entries: + print_warning("No plugins found") + return + + self._display_catalog_entries(entries) + + def run_search( + self, + query: str, + *, + tap_alias: str | None = None, + refresh: bool = False, + include_incompatible: bool = False, + ) -> None: + """Search plugins from a tap catalog.""" + tap = self._get_tap_or_exit(tap_alias) + try: + entries = self.catalog_service.search_entries( + query, + tap.alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + except Exception as e: + print_error(f"Failed to search plugin catalog: {e}") + raise typer.Exit(code=1) + + print_header("Data Designer Plugin Search") + print_info(f"Tap: {tap.alias} ({tap.url})") + print_info(f"Query: {query}") + console.print() + + if not entries: + print_warning("No matching plugins found") + return + + self._display_catalog_entries(entries) + + def run_info( + self, + plugin_name: str, + *, + tap_alias: str | None = None, + refresh: bool = False, + ) -> None: + """Show full metadata for one plugin.""" + tap = self._get_tap_or_exit(tap_alias) + entry = self._get_entry_or_exit(plugin_name, tap.alias, refresh=refresh) + compatibility = self.catalog_service.evaluate_compatibility(entry) + + print_header(f"Plugin: {entry.name}") + print_info(f"Tap: {tap.alias} ({tap.url})") + self._display_compatibility(compatibility) + + try: + plan = self.install_service.build_install_plan(entry, tap) + console.print(f" Install command: [bold]{shlex.join(plan.command)}[/bold]") + except ValueError as e: + print_warning(str(e)) + + console.print() + display_config_preview(entry.model_dump(mode="json", exclude_none=True), "Plugin Metadata") + + def run_install( + self, + plugin_name: str, + *, + tap_alias: str | None = None, + refresh: bool = False, + manager: str = "auto", + yes: bool = False, + dry_run: bool = False, + force: bool = False, + ) -> None: + """Install one plugin from a catalog entry.""" + tap = self._get_tap_or_exit(tap_alias) + entry = self._get_entry_or_exit(plugin_name, tap.alias, refresh=refresh, include_incompatible=force) + compatibility = self.catalog_service.evaluate_compatibility(entry) + + if not compatibility.is_compatible and not force: + print_error(f"Plugin {entry.name!r} is not compatible with this environment") + for reason in compatibility.reasons: + console.print(f" - {reason}") + raise typer.Exit(code=1) + + try: + plan = self.install_service.build_install_plan(entry, tap, manager=manager) + except ValueError as e: + print_error(f"Failed to build plugin install plan: {e}") + raise typer.Exit(code=1) + + print_header("Install Data Designer Plugin") + console.print(f" Plugin: [bold]{entry.name}[/bold]") + console.print(f" Tap: [bold]{tap.alias}[/bold] ({tap.url})") + console.print(f" Source: [bold]{plan.source_description}[/bold]") + console.print(f" Command: [bold]{shlex.join(plan.command)}[/bold]") + self._display_compatibility(compatibility) + + if not tap.trusted: + print_warning( + "This tap is not marked trusted. Plugin installation executes Python package code from the source above." + ) + + if dry_run: + print_info("Dry run complete; no changes made") + return + + if not yes and not confirm_action("Install this plugin into the current Python environment?", default=False): + print_info("No changes made") + return + + try: + self.install_service.install(plan) + except RuntimeError as e: + print_error(str(e)) + raise typer.Exit(code=1) + + if self.install_service.verify_entry_point(entry): + print_success(f"Plugin {entry.name!r} installed and discovered") + else: + print_warning( + f"Plugin {entry.name!r} was installed, but Data Designer did not discover its entry point. " + "Restart the shell or check the package entry point metadata." + ) + + def run_installed(self) -> None: + """List plugins currently discoverable through runtime entry points.""" + print_header("Installed Data Designer Plugins") + installed_plugins = self.catalog_service.list_installed_plugins() + if not installed_plugins: + print_warning("No installed Data Designer plugins were discovered") + return + self._display_installed_plugins(installed_plugins) + + def run_taps_list(self) -> None: + """List configured plugin taps.""" + print_header("Data Designer Plugin Taps") + taps = self.catalog_service.list_taps() + table = Table(title="Plugin Taps", border_style=NordColor.NORD8.value) + table.add_column("Alias", style=NordColor.NORD14.value, no_wrap=True) + table.add_column("URL", style=NordColor.NORD4.value) + table.add_column("Trusted", style=NordColor.NORD13.value, justify="center") + table.add_column("Cache TTL", style=NordColor.NORD9.value, justify="right") + + for tap in taps: + table.add_row( + tap.alias, + tap.url, + "yes" if tap.trusted else "no", + f"{tap.cache_ttl_seconds}s", + ) + console.print(table) + + def run_taps_add( + self, + *, + alias: str, + url: str, + trusted: bool, + cache_ttl_seconds: int, + ) -> None: + """Add a plugin tap alias.""" + try: + tap = self.catalog_service.add_tap( + alias, + url, + trusted=trusted, + cache_ttl_seconds=cache_ttl_seconds, + ) + except Exception as e: + print_error(f"Failed to add plugin tap: {e}") + raise typer.Exit(code=1) + + print_success(f"Plugin tap {tap.alias!r} added") + print_info(f"Catalog: {tap.url}") + + def run_taps_remove(self, *, alias: str) -> None: + """Remove a plugin tap alias.""" + try: + self.catalog_service.remove_tap(alias) + except Exception as e: + print_error(f"Failed to remove plugin tap: {e}") + raise typer.Exit(code=1) + print_success(f"Plugin tap {alias!r} removed") + + def _get_tap_or_exit(self, tap_alias: str | None) -> PluginTapConfig: + try: + return self.catalog_service.get_tap(tap_alias or DEFAULT_PLUGIN_TAP_ALIAS) + except ValueError as e: + print_error(str(e)) + raise typer.Exit(code=1) + + def _list_entries_or_exit( + self, + tap_alias: str, + *, + refresh: bool, + include_incompatible: bool, + ) -> list[PluginCatalogEntry]: + try: + return self.catalog_service.list_entries( + tap_alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + except Exception as e: + print_error(f"Failed to load plugin catalog: {e}") + raise typer.Exit(code=1) + + def _get_entry_or_exit( + self, + plugin_name: str, + tap_alias: str, + *, + refresh: bool, + include_incompatible: bool = True, + ) -> PluginCatalogEntry: + try: + return self.catalog_service.get_entry( + plugin_name, + tap_alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + except Exception as e: + print_error(str(e)) + raise typer.Exit(code=1) + + def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: + table = Table(title="Catalog Plugins", border_style=NordColor.NORD8.value) + table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) + table.add_column("Type", style=NordColor.NORD9.value, no_wrap=True) + table.add_column("Package", style=NordColor.NORD4.value) + table.add_column("Version", style=NordColor.NORD15.value, no_wrap=True) + table.add_column("Compatible", style=NordColor.NORD13.value, no_wrap=True) + table.add_column("Docs", style=NordColor.NORD7.value) + + for entry in entries: + compatibility = self.catalog_service.evaluate_compatibility(entry) + docs_url = entry.docs.url if entry.docs is not None and entry.docs.url is not None else "" + table.add_row( + entry.name, + entry.plugin_type.value, + entry.package.name, + entry.package.version or "", + "yes" if compatibility.is_compatible else "no", + docs_url, + ) + console.print(table) + + @staticmethod + def _display_installed_plugins(installed_plugins: list[InstalledPluginInfo]) -> None: + table = Table(title="Installed Plugins", border_style=NordColor.NORD8.value) + table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) + table.add_column("Type", style=NordColor.NORD9.value, no_wrap=True) + table.add_column("Config", style=NordColor.NORD4.value) + table.add_column("Implementation", style=NordColor.NORD7.value) + + for plugin in installed_plugins: + table.add_row( + plugin.name, + plugin.plugin_type.value, + plugin.config_qualified_name, + plugin.impl_qualified_name, + ) + console.print(table) + + @staticmethod + def _display_compatibility(compatibility: CompatibilityResult) -> None: + if compatibility.is_compatible: + console.print(" Compatibility: [bold green]compatible[/bold green]") + return + + console.print(" Compatibility: [bold yellow]not compatible[/bold yellow]") + for reason in compatibility.reasons: + console.print(f" - {reason}") diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index 757ff8073..f85e78df7 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -141,6 +141,79 @@ def _is_version_request(args: list[str]) -> bool: no_args_is_help=True, ) +# Create plugins command group +plugins_app = typer.Typer( + name="plugins", + help="Discover and install Data Designer plugins from tap catalogs", + cls=create_lazy_typer_group( + { + "list": { + "module": f"{_CMD}.plugins", + "attr": "list_command", + "help": "List plugins from a tap catalog", + }, + "search": { + "module": f"{_CMD}.plugins", + "attr": "search_command", + "help": "Search plugins from a tap catalog", + }, + "info": { + "module": f"{_CMD}.plugins", + "attr": "info_command", + "help": "Show plugin metadata and install plan", + }, + "install": { + "module": f"{_CMD}.plugins", + "attr": "install_command", + "help": "Install a plugin package and verify discovery", + }, + "installed": { + "module": f"{_CMD}.plugins", + "attr": "installed_command", + "help": "List installed runtime plugins", + }, + } + ), + no_args_is_help=True, +) + + +@plugins_app.callback() +def plugins_callback( + tap: str | None = typer.Option( + None, + "--tap", + help="Plugin tap alias to use for catalog commands.", + ), +) -> None: + _ = tap + + +plugin_taps_app = typer.Typer( + name="taps", + help="Manage plugin tap aliases", + cls=create_lazy_typer_group( + { + "list": { + "module": f"{_CMD}.plugins", + "attr": "taps_list_command", + "help": "List configured plugin taps", + }, + "add": { + "module": f"{_CMD}.plugins", + "attr": "taps_add_command", + "help": "Add a plugin tap alias", + }, + "remove": { + "module": f"{_CMD}.plugins", + "attr": "taps_remove_command", + "help": "Remove a plugin tap alias", + }, + } + ), + no_args_is_help=True, +) + _AGENT_CMD = f"{_CMD}.agent" @@ -167,10 +240,12 @@ def _build_agent_lazy_group(prefix: str) -> dict[str, dict[str, str]]: ) agent_app.add_typer(agent_state_app, name="state") +plugins_app.add_typer(plugin_taps_app, name="taps") # Add setup command groups app.add_typer(config_app, name="config", rich_help_panel="Setup") app.add_typer(download_app, name="download", rich_help_panel="Setup") +app.add_typer(plugins_app, name="plugins", rich_help_panel="Setup") app.add_typer(agent_app, name="agent", rich_help_panel="Agent") From 1239643fd0014ff40b9043b566712175a6b80d1d Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 20:29:12 +0000 Subject: [PATCH 03/34] test(cli): cover plugin catalog workflows Add regression coverage for tap caching, catalog compatibility, installer command generation, local path resolution, and Typer command delegation. Refs #617 --- .../cli/commands/test_plugins_command.py | 90 +++++++ .../test_plugin_tap_repository.py | 113 +++++++++ .../services/test_plugin_catalog_service.py | 221 ++++++++++++++++++ .../services/test_plugin_install_service.py | 135 +++++++++++ 4 files changed, 559 insertions(+) create mode 100644 packages/data-designer/tests/cli/commands/test_plugins_command.py create mode 100644 packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py create mode 100644 packages/data-designer/tests/cli/services/test_plugin_catalog_service.py create mode 100644 packages/data-designer/tests/cli/services/test_plugin_install_service.py diff --git a/packages/data-designer/tests/cli/commands/test_plugins_command.py b/packages/data-designer/tests/cli/commands/test_plugins_command.py new file mode 100644 index 000000000..c895d360d --- /dev/null +++ b/packages/data-designer/tests/cli/commands/test_plugins_command.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from data_designer.cli.main import app + +runner = CliRunner() + + +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_list_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke(app, ["plugins", "--tap", "research", "list", "--refresh", "--include-incompatible"]) + + assert result.exit_code == 0 + mock_ctrl.run_list.assert_called_once_with( + tap_alias="research", + refresh=True, + include_incompatible=True, + ) + + +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_search_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke(app, ["plugins", "search", "github", "--tap", "research"]) + + assert result.exit_code == 0 + mock_ctrl.run_search.assert_called_once_with( + "github", + tap_alias="research", + refresh=False, + include_incompatible=False, + ) + + +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke(app, ["plugins", "install", "text-transform", "--manager", "pip", "--yes", "--dry-run"]) + + assert result.exit_code == 0 + mock_ctrl.run_install.assert_called_once_with( + "text-transform", + tap_alias=None, + refresh=False, + manager="pip", + yes=True, + dry_run=True, + force=False, + ) + + +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_taps_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke( + app, + [ + "plugins", + "taps", + "add", + "research", + "https://github.com/acme/dd-plugins", + "--trusted", + "--cache-ttl-seconds", + "60", + ], + ) + + assert result.exit_code == 0 + mock_ctrl.run_taps_add.assert_called_once_with( + alias="research", + url="https://github.com/acme/dd-plugins", + trusted=True, + cache_ttl_seconds=60, + ) diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py new file mode 100644 index 000000000..774bf6a6a --- /dev/null +++ b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from data_designer.cli.plugin_catalog import DEFAULT_PLUGIN_TAP_ALIAS, PluginCatalogError +from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository, normalize_tap_location + + +def test_repository_includes_default_nvidia_tap(tmp_path: Path) -> None: + repository = PluginTapRepository(tmp_path) + + taps = repository.list_taps() + + assert [tap.alias for tap in taps] == [DEFAULT_PLUGIN_TAP_ALIAS] + assert taps[0].trusted is True + + +def test_add_tap_normalizes_github_repository_url(tmp_path: Path) -> None: + repository = PluginTapRepository(tmp_path) + + tap = repository.add_tap("research", "https://github.com/acme/dd-plugins") + + assert tap.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json" + assert repository.get_tap("research") == tap + + +def test_normalize_local_tap_directory() -> None: + normalized = normalize_tap_location("~/plugins") + + assert normalized.endswith("/plugins/catalog/plugins.json") + + +def test_load_catalog_uses_cache_when_source_is_unavailable(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + first_catalog = repository.load_catalog("local") + catalog_path.unlink() + cached_catalog = repository.load_catalog("local") + + assert first_catalog.plugins[0].name == "text-transform" + assert cached_catalog.plugins[0].name == "text-transform" + + +def test_load_catalog_with_zero_cache_ttl_refreshes_source(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path, plugin_name="text-transform") + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path), cache_ttl_seconds=0) + + first_catalog = repository.load_catalog("local") + catalog_path.write_text(json.dumps(_catalog_payload(plugin_name="fresh-transform"))) + refreshed_catalog = repository.load_catalog("local") + + assert first_catalog.plugins[0].name == "text-transform" + assert refreshed_catalog.plugins[0].name == "fresh-transform" + + +def test_load_catalog_rejects_unsupported_schema_version(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path, schema_version=999) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="Unsupported plugin catalog schema_version"): + repository.load_catalog("local", refresh=True) + + +def _write_catalog(tmp_path: Path, *, schema_version: int = 2, plugin_name: str = "text-transform") -> Path: + catalog_dir = tmp_path / "catalog" + catalog_dir.mkdir() + catalog_path = catalog_dir / "plugins.json" + catalog_path.write_text(json.dumps(_catalog_payload(schema_version=schema_version, plugin_name=plugin_name))) + return catalog_path + + +def _catalog_payload(*, schema_version: int = 2, plugin_name: str = "text-transform") -> dict: + return { + "schema_version": schema_version, + "plugins": [ + { + "name": plugin_name, + "plugin_type": "processor", + "description": "Transform text records", + "package": { + "name": "data-designer-text-transform", + "version": "0.1.0", + "path": "plugins/data-designer-text-transform", + }, + "entry_point": { + "group": "data_designer.plugins", + "name": plugin_name, + "value": "data_designer_text_transform.plugin:plugin", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": {"specifier": ">=0.5.7"}, + }, + "source": { + "type": "pypi", + "package": "data-designer-text-transform", + }, + "docs": { + "url": "https://example.com/text-transform", + }, + }, + ], + } diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py new file mode 100644 index 000000000..6f2c50257 --- /dev/null +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import Mock, patch + +from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository +from data_designer.cli.services.plugin_catalog_service import PluginCatalogService +from data_designer.plugins.plugin import PluginType + + +def test_list_entries_filters_incompatible_plugins_by_default(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + entries = service.list_entries("local") + all_entries = service.list_entries("local", include_incompatible=True) + + assert [entry.name for entry in entries] == [ + "compatible-plugin", + "shared-column", + "shared-processor", + "versioned-plugin", + "versioned-plugin", + ] + assert [entry.name for entry in all_entries] == [ + "compatible-plugin", + "future-plugin", + "shared-column", + "shared-processor", + "versioned-plugin", + "versioned-plugin", + "versioned-plugin", + ] + + +def test_search_entries_matches_name_type_package_and_tags(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + name_matches = service.search_entries("compatible", "local") + tag_matches = service.search_entries("github", "local") + type_matches = service.search_entries("seed-reader", "local") + + assert [entry.name for entry in name_matches] == ["compatible-plugin"] + assert [entry.name for entry in tag_matches] == ["compatible-plugin"] + assert [entry.name for entry in type_matches] == ["compatible-plugin"] + + +def test_evaluate_compatibility_reports_data_designer_constraint(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + entry = service.get_entry("future-plugin", "local") + + result = service.evaluate_compatibility(entry) + + assert result.is_compatible is False + assert result.reasons == ["Data Designer 0.5.7 does not satisfy >=99.0"] + + +def test_evaluate_compatibility_accepts_local_dev_version_above_lower_bound(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService( + repository, + python_version="3.11.0", + data_designer_version="0.5.10.dev18+604fdd96", + ) + entry = service.get_entry("compatible-plugin", "local") + + result = service.evaluate_compatibility(entry) + + assert result.is_compatible is True + assert result.reasons == [] + + +def test_get_entry_returns_newest_compatible_version(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + entry = service.get_entry("versioned-plugin", "local", include_incompatible=False) + newest_entry = service.get_entry("versioned-plugin", "local", include_incompatible=True) + + assert entry.package.version == "0.2.0" + assert newest_entry.package.version == "99.0.0" + + +def test_group_entries_by_package_groups_multi_plugin_packages(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + entries = service.list_entries("local", include_incompatible=True) + + grouped_entries = service.group_entries_by_package(entries) + + assert [entry.name for entry in grouped_entries["data-designer-shared-package"]] == [ + "shared-column", + "shared-processor", + ] + + +@patch("data_designer.cli.services.plugin_catalog_service.PluginRegistry") +def test_list_installed_plugins_uses_runtime_registry(mock_registry_cls: Mock, tmp_path: Path) -> None: + plugin = Mock() + plugin.name = "installed-plugin" + plugin.plugin_type = PluginType.PROCESSOR + plugin.config_qualified_name = "pkg.config.Config" + plugin.impl_qualified_name = "pkg.impl.Impl" + mock_registry = Mock() + mock_registry.get_plugins.side_effect = lambda plugin_type: [plugin] if plugin_type == PluginType.PROCESSOR else [] + mock_registry_cls.return_value = mock_registry + service = PluginCatalogService(PluginTapRepository(tmp_path)) + + installed = service.list_installed_plugins() + + assert len(installed) == 1 + assert installed[0].name == "installed-plugin" + assert installed[0].plugin_type == PluginType.PROCESSOR + + +def _repository_with_catalog(tmp_path: Path) -> PluginTapRepository: + catalog_path = tmp_path / "plugins.json" + catalog_path.write_text(json.dumps(_catalog_payload())) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + return repository + + +def _catalog_payload() -> dict: + return { + "schema_version": 2, + "plugins": [ + _entry( + name="compatible-plugin", + plugin_type="seed-reader", + package_name="data-designer-compatible-plugin", + data_designer_specifier=">=0.5.7", + tags=["github", "repository"], + ), + _entry( + name="future-plugin", + plugin_type="processor", + package_name="data-designer-future-plugin", + data_designer_specifier=">=99.0", + tags=["future"], + ), + _entry( + name="versioned-plugin", + plugin_type="processor", + package_name="data-designer-versioned-plugin", + data_designer_specifier=">=0.5.7", + package_version="0.1.0", + tags=["versioned"], + ), + _entry( + name="versioned-plugin", + plugin_type="processor", + package_name="data-designer-versioned-plugin", + data_designer_specifier=">=0.5.7", + package_version="0.2.0", + tags=["versioned"], + ), + _entry( + name="versioned-plugin", + plugin_type="processor", + package_name="data-designer-versioned-plugin", + data_designer_specifier=">=99.0", + package_version="99.0.0", + tags=["versioned"], + ), + _entry( + name="shared-column", + plugin_type="column-generator", + package_name="data-designer-shared-package", + data_designer_specifier=">=0.5.7", + tags=["shared"], + ), + _entry( + name="shared-processor", + plugin_type="processor", + package_name="data-designer-shared-package", + data_designer_specifier=">=0.5.7", + tags=["shared"], + ), + ], + } + + +def _entry( + *, + name: str, + plugin_type: str, + package_name: str, + data_designer_specifier: str, + tags: list[str], + package_version: str = "0.1.0", +) -> dict: + return { + "name": name, + "plugin_type": plugin_type, + "description": f"{name} description", + "package": { + "name": package_name, + "version": package_version, + }, + "entry_point": { + "group": "data_designer.plugins", + "name": name, + "value": f"{package_name.replace('-', '_')}.plugin:plugin", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": {"specifier": data_designer_specifier}, + }, + "source": { + "type": "pypi", + "package": package_name, + }, + "tags": tags, + } diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py new file mode 100644 index 000000000..b75e1a557 --- /dev/null +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from data_designer.cli.plugin_catalog import PluginCatalogEntry, PluginTapConfig +from data_designer.cli.services.plugin_install_service import PluginInstallService + + +def test_build_pypi_install_plan_uses_exact_catalog_version() -> None: + entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) + tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_install_plan(entry, tap, manager="pip") + + assert plan.command == [ + sys.executable, + "-m", + "pip", + "install", + "data-designer-text-transform==0.1.0", + ] + assert plan.source_description == "data-designer-text-transform==0.1.0" + + +def test_build_git_install_plan_includes_ref_and_subdirectory() -> None: + entry = _entry( + source={ + "type": "git", + "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", + "ref": "data-designer-text-transform/v0.1.0", + "subdirectory": "plugins/data-designer-text-transform", + } + ) + tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_install_plan(entry, tap, manager="pip") + + assert plan.command[-1] == ( + "git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git" + "@data-designer-text-transform/v0.1.0" + "#subdirectory=plugins/data-designer-text-transform" + ) + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_uv_install_plan_targets_current_python(mock_which: Mock) -> None: + entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) + tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_install_plan(entry, tap, manager="uv") + + assert plan.command[:5] == ["uv", "pip", "install", "--python", sys.executable] + + +def test_build_path_install_plan_resolves_relative_path_from_local_tap_root(tmp_path: Path) -> None: + entry = _entry(source={"type": "path", "editable": True}) + tap = PluginTapConfig(alias="local", url=str(tmp_path / "catalog" / "plugins.json")) + service = PluginInstallService() + + plan = service.build_install_plan(entry, tap, manager="pip") + + assert plan.command == [ + sys.executable, + "-m", + "pip", + "install", + "-e", + str(tmp_path / "plugins" / "data-designer-text-transform"), + ] + + +def test_build_install_plan_requires_source() -> None: + entry = _entry(source=None) + tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + with pytest.raises(ValueError, match="does not declare a source"): + service.build_install_plan(entry, tap, manager="pip") + + +def test_install_raises_when_runner_fails() -> None: + service = PluginInstallService(runner=lambda command: 2) + entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) + tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + plan = service.build_install_plan(entry, tap, manager="pip") + + with pytest.raises(RuntimeError, match="status 2"): + service.install(plan) + + +@patch("data_designer.cli.services.plugin_install_service.PluginRegistry") +def test_verify_entry_point_resets_and_checks_runtime_registry(mock_registry_cls: Mock) -> None: + entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) + mock_registry = Mock() + mock_registry.plugin_exists.return_value = True + mock_registry_cls.return_value = mock_registry + service = PluginInstallService() + + assert service.verify_entry_point(entry) is True + mock_registry_cls.reset.assert_called_once_with() + mock_registry.plugin_exists.assert_called_once_with("text-transform") + + +def _entry(source: dict | None) -> PluginCatalogEntry: + payload = { + "name": "text-transform", + "plugin_type": "processor", + "description": "Transform text records", + "package": { + "name": "data-designer-text-transform", + "version": "0.1.0", + "path": "plugins/data-designer-text-transform", + }, + "entry_point": { + "group": "data_designer.plugins", + "name": "text-transform", + "value": "data_designer_text_transform.plugin:plugin", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": {"specifier": ">=0.5.7"}, + }, + "source": source, + } + return PluginCatalogEntry.model_validate(payload) From bf9f522926b6f045427c7b45995ce0838208c4f6 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 20:43:47 +0000 Subject: [PATCH 04/34] fix(cli): align plugin taps with schema v2 Validate tap catalogs against the schema v2 contract used by NVIDIA-NeMo/DataDesignerPlugins#36, including source union fields, docs URLs, package paths, compatibility metadata, and unique runtime plugin names. Derive Git install targets as package-qualified PEP 508 direct references so git tap entries install the package described by the catalog source metadata. Refs #617 --- .../src/data_designer/cli/plugin_catalog.py | 339 +++++++++++++++++- .../cli/repositories/plugin_tap_repository.py | 12 +- .../cli/services/plugin_catalog_service.py | 7 - .../cli/services/plugin_install_service.py | 25 +- .../test_plugin_tap_repository.py | 171 +++++++-- .../services/test_plugin_catalog_service.py | 62 +--- .../services/test_plugin_install_service.py | 33 +- 7 files changed, 530 insertions(+), 119 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index 4cbebfe37..f3b5564f6 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -3,8 +3,14 @@ from __future__ import annotations +import re from dataclasses import dataclass +from urllib.parse import urlparse +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import InvalidName, canonicalize_name +from packaging.version import InvalidVersion, Version from pydantic import BaseModel, ConfigDict, Field from data_designer.plugins.plugin import PluginType @@ -15,8 +21,31 @@ PLUGIN_TAP_CACHE_DIR_NAME = "plugin-tap-cache" PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS = 24 * 60 * 60 MAX_PLUGIN_CATALOG_SIZE_BYTES = 1 * 1024 * 1024 -SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS = {1, 2} +PLUGIN_CATALOG_SCHEMA_VERSION = 2 +SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS = {PLUGIN_CATALOG_SCHEMA_VERSION} PLUGIN_TAP_ALIAS_PATTERN = r"^[A-Za-z0-9_.-]+$" +DATA_DESIGNER_DISTRIBUTION_NAME = "data-designer" +PLUGIN_ENTRY_POINT_GROUP = "data_designer.plugins" +CATALOG_DOCUMENT_KEYS = {"plugins", "schema_version"} +CATALOG_PLUGIN_KEYS = { + "compatibility", + "description", + "docs", + "entry_point", + "name", + "package", + "plugin_type", + "source", +} +CATALOG_PACKAGE_KEYS = {"name", "path", "version"} +CATALOG_ENTRY_POINT_KEYS = {"group", "name", "value"} +CATALOG_COMPATIBILITY_KEYS = {"data_designer", "python"} +CATALOG_PYTHON_COMPATIBILITY_KEYS = {"specifier"} +CATALOG_DATA_DESIGNER_COMPATIBILITY_KEYS = {"marker", "requirement", "specifier"} +CATALOG_DOCS_KEYS = {"url"} +SUPPORTED_PLUGIN_TYPE_VALUES = {plugin_type.value for plugin_type in PluginType} +PACKAGE_PATH_ROOT = "plugins" +PACKAGE_PATH_SEGMENT_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") class PluginCatalogError(ValueError): @@ -28,6 +57,7 @@ class PluginCompatibilityTarget(BaseModel): model_config = ConfigDict(extra="allow") + requirement: str | None = None specifier: str | None = None marker: str | None = None @@ -72,7 +102,7 @@ class PluginSourceInfo(BaseModel): ref: str | None = None path: str | None = None subdirectory: str | None = None - editable: bool = False + editable: bool | None = None class PluginDocsInfo(BaseModel): @@ -96,9 +126,6 @@ class PluginCatalogEntry(BaseModel): compatibility: PluginCompatibility | None = None source: PluginSourceInfo | None = None docs: PluginDocsInfo | None = None - tags: list[str] = Field(default_factory=list) - maintainers: list[str] = Field(default_factory=list) - release_notes_url: str | None = None class PluginCatalog(BaseModel): @@ -154,3 +181,305 @@ class InstalledPluginInfo: plugin_type: PluginType config_qualified_name: str impl_qualified_name: str + + +def validate_plugin_catalog_payload(payload: object, *, source: str) -> None: + """Validate a decoded plugin tap catalog against the schema v2 contract.""" + try: + _validate_plugin_catalog_payload(payload) + except PluginCatalogError as e: + raise PluginCatalogError(f"Invalid plugin catalog at {source!r}: {e}") from e + + +def _validate_plugin_catalog_payload(payload: object) -> None: + catalog = _required_catalog_object("catalog document", payload, CATALOG_DOCUMENT_KEYS) + schema_version = catalog["schema_version"] + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or schema_version != PLUGIN_CATALOG_SCHEMA_VERSION + ): + raise PluginCatalogError( + f"unsupported catalog schema_version {schema_version!r}; expected {PLUGIN_CATALOG_SCHEMA_VERSION}" + ) + + plugins = catalog["plugins"] + if not isinstance(plugins, list): + raise PluginCatalogError("catalog document has invalid plugins; expected a list") + + runtime_names: dict[str, tuple[str, str]] = {} + for index, raw_plugin in enumerate(plugins): + package_name, plugin_name, entry_point_name = _validate_catalog_plugin(raw_plugin, index) + previous = runtime_names.get(plugin_name) + if previous is not None: + previous_package, previous_entry_point = previous + raise PluginCatalogError( + f"duplicate runtime plugin name {plugin_name!r} from " + f"{previous_package!r} entry point {previous_entry_point!r} and " + f"{package_name!r} entry point {entry_point_name!r}" + ) + runtime_names[plugin_name] = (package_name, entry_point_name) + + +def _validate_catalog_plugin(raw_plugin: object, index: int) -> tuple[str, str, str]: + context = f"catalog plugins[{index}]" + plugin = _required_catalog_object(context, raw_plugin, CATALOG_PLUGIN_KEYS) + package = _required_catalog_object(f"{context}.package", plugin["package"], CATALOG_PACKAGE_KEYS) + entry_point = _required_catalog_object( + f"{context}.entry_point", + plugin["entry_point"], + CATALOG_ENTRY_POINT_KEYS, + ) + compatibility = _required_catalog_object( + f"{context}.compatibility", + plugin["compatibility"], + CATALOG_COMPATIBILITY_KEYS, + ) + python_compatibility = _required_catalog_object( + f"{context}.compatibility.python", + compatibility["python"], + CATALOG_PYTHON_COMPATIBILITY_KEYS, + ) + data_designer_compatibility = _required_catalog_object( + f"{context}.compatibility.data_designer", + compatibility["data_designer"], + CATALOG_DATA_DESIGNER_COMPATIBILITY_KEYS, + ) + source = _required_catalog_object(f"{context}.source", plugin["source"]) + docs = _required_catalog_object(f"{context}.docs", plugin["docs"], CATALOG_DOCS_KEYS) + + package_name = _catalog_package_name(f"{context}.package.name", package["name"]) + _catalog_version(package_name, f"{context}.package.version", package["version"]) + _validate_package_path(package_name, _required_catalog_string(f"{context}.package.path", package["path"])) + + plugin_type = _required_catalog_string(f"{context}.plugin_type", plugin["plugin_type"]) + if plugin_type not in SUPPORTED_PLUGIN_TYPE_VALUES: + raise PluginCatalogError( + f"{context}.plugin_type {plugin_type!r} is invalid; expected one of " + f"{_format_catalog_choices(SUPPORTED_PLUGIN_TYPE_VALUES)}" + ) + + plugin_name = _required_catalog_string(f"{context}.name", plugin["name"]) + entry_point_group = _required_catalog_string(f"{context}.entry_point.group", entry_point["group"]) + if entry_point_group != PLUGIN_ENTRY_POINT_GROUP: + raise PluginCatalogError( + f"{context}.entry_point.group {entry_point_group!r} is invalid; expected {PLUGIN_ENTRY_POINT_GROUP!r}" + ) + entry_point_name = _required_catalog_string(f"{context}.entry_point.name", entry_point["name"]) + _required_catalog_string(f"{context}.entry_point.value", entry_point["value"]) + _required_catalog_string(f"{context}.description", plugin["description"]) + _catalog_version_specifier( + package_name, + f"{context}.compatibility.python.specifier", + python_compatibility["specifier"], + ) + _catalog_data_designer_compatibility( + package_name, + f"{context}.compatibility.data_designer", + data_designer_compatibility, + ) + _validate_source_metadata(package_name, source) + _catalog_http_url(f"{context}.docs.url", docs["url"]) + return package_name, plugin_name, entry_point_name + + +def _required_catalog_object( + context: str, + value: object, + expected_keys: set[str] | None = None, +) -> dict[str, object]: + if not isinstance(value, dict): + raise PluginCatalogError(f"{context} is invalid; expected an object") + if expected_keys is not None: + _validate_catalog_object_keys(context, value, expected_keys) + return value + + +def _validate_catalog_object_keys(context: str, value: dict[str, object], expected_keys: set[str]) -> None: + keys = set(value) + if keys != expected_keys: + raise PluginCatalogError( + f"{context} has invalid fields; expected {{{_format_catalog_keys(expected_keys)}}}, " + f"got {{{_format_catalog_keys(keys)}}}" + ) + + +def _required_catalog_string(context: str, value: object) -> str: + if not isinstance(value, str) or not value: + raise PluginCatalogError(f"{context} is invalid; expected a non-empty string") + return value + + +def _required_catalog_nullable_string(context: str, value: object) -> str | None: + if value is None: + return None + if isinstance(value, str): + return value + raise PluginCatalogError(f"{context} is invalid; expected a string or null") + + +def _catalog_package_name(context: str, value: object) -> str: + package_name = _required_catalog_string(context, value) + try: + canonicalize_name(package_name, validate=True) + except InvalidName as e: + raise PluginCatalogError(f"{context} {package_name!r} is invalid; expected a valid package name") from e + return package_name + + +def _catalog_version(package_name: str, context: str, value: object) -> str: + raw_version = _required_catalog_string(context, value) + try: + Version(raw_version) + except InvalidVersion as e: + raise PluginCatalogError(f"package {package_name!r} has invalid {context} {raw_version!r}: {e}") from e + return raw_version + + +def _catalog_version_specifier(package_name: str, context: str, value: object) -> str: + raw_specifier = _required_catalog_string(context, value) + try: + specifier = SpecifierSet(raw_specifier) + except InvalidSpecifier as e: + raise PluginCatalogError(f"package {package_name!r} has invalid {context} {raw_specifier!r}: {e}") from e + if not str(specifier): + raise PluginCatalogError(f"package {package_name!r} has invalid {context}; expected at least one specifier") + return str(specifier) + + +def _catalog_data_designer_compatibility( + package_name: str, + context: str, + compatibility: dict[str, object], +) -> None: + requirement_text = _required_catalog_string(f"{context}.requirement", compatibility["requirement"]) + try: + requirement = Requirement(requirement_text) + except InvalidRequirement as e: + raise PluginCatalogError( + f"package {package_name!r} has invalid {context}.requirement {requirement_text!r}: {e}" + ) from e + if canonicalize_name(requirement.name) != DATA_DESIGNER_DISTRIBUTION_NAME: + raise PluginCatalogError( + f"package {package_name!r} has invalid {context}.requirement {requirement_text!r}; " + f"expected a {DATA_DESIGNER_DISTRIBUTION_NAME!r} requirement" + ) + if not requirement.specifier: + raise PluginCatalogError(f"package {package_name!r} has invalid {context}.requirement; expected a specifier") + + specifier = _catalog_version_specifier(package_name, f"{context}.specifier", compatibility["specifier"]) + if specifier != str(requirement.specifier): + raise PluginCatalogError( + f"package {package_name!r} has invalid {context}.specifier {specifier!r}; " + f"expected {str(requirement.specifier)!r} from requirement" + ) + + marker = _required_catalog_nullable_string(f"{context}.marker", compatibility["marker"]) + expected_marker = str(requirement.marker) if requirement.marker is not None else None + if marker != expected_marker: + raise PluginCatalogError( + f"package {package_name!r} has invalid {context}.marker {marker!r}; expected {expected_marker!r}" + ) + + +def _validate_source_metadata(package_name: str, source: dict[str, object]) -> None: + source_type = source.get("type") + if source_type == "pypi": + _validate_pypi_source_metadata(package_name, source) + return + if source_type == "git": + _validate_git_source_metadata(package_name, source) + return + if source_type == "path": + _validate_path_source_metadata(package_name, source) + return + raise PluginCatalogError( + f"package {package_name!r} has invalid source.type {source_type!r}; expected one of 'pypi', 'git', or 'path'" + ) + + +def _validate_pypi_source_metadata(package_name: str, source: dict[str, object]) -> None: + _validate_source_keys(package_name, source, "pypi", {"type", "package"}) + source_package = _required_source_string(package_name, source, "pypi", "package") + if source_package != package_name: + raise PluginCatalogError( + f"package {package_name!r} has invalid pypi source package {source_package!r}; " + "expected the source package to match package.name" + ) + + +def _validate_git_source_metadata(package_name: str, source: dict[str, object]) -> None: + _validate_source_keys(package_name, source, "git", {"type", "url", "ref", "subdirectory"}) + url = _required_source_string(package_name, source, "git", "url") + _required_source_string(package_name, source, "git", "ref") + subdirectory = _required_source_string(package_name, source, "git", "subdirectory") + _validate_package_path(package_name, subdirectory) + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise PluginCatalogError( + f"package {package_name!r} has invalid git source url {url!r}; expected an absolute HTTP(S) URL" + ) + + +def _validate_path_source_metadata(package_name: str, source: dict[str, object]) -> None: + _validate_source_keys(package_name, source, "path", {"type", "path", "editable"}) + path = _required_source_string(package_name, source, "path", "path") + _validate_package_path(package_name, path) + editable = source.get("editable") + if not isinstance(editable, bool): + raise PluginCatalogError(f"package {package_name!r} has invalid path source field 'editable'; expected a bool") + + +def _validate_source_keys( + package_name: str, + source: dict[str, object], + source_type: str, + expected_keys: set[str], +) -> None: + keys = set(source) + if keys != expected_keys: + raise PluginCatalogError( + f"package {package_name!r} has invalid {source_type!r} source fields; " + f"expected {{{_format_catalog_keys(expected_keys)}}}, got {{{_format_catalog_keys(keys)}}}" + ) + + +def _required_source_string(package_name: str, source: dict[str, object], source_type: str, key: str) -> str: + value = source.get(key) + if not isinstance(value, str) or not value: + raise PluginCatalogError( + f"package {package_name!r} has invalid {source_type!r} source field {key!r}; expected a non-empty string" + ) + return value + + +def _validate_package_path(package_name: str, value: str) -> None: + parts = value.split("/") + if ( + "\\" in value + or value.startswith("/") + or len(parts) < 2 + or parts[0] != PACKAGE_PATH_ROOT + or any(part in {"", ".", ".."} for part in parts) + or not all(PACKAGE_PATH_SEGMENT_PATTERN.fullmatch(part) for part in parts[1:]) + ): + raise PluginCatalogError( + f"package {package_name!r} has invalid package path {value!r}; " + f"expected a normalized repository-relative path under {PACKAGE_PATH_ROOT!r}" + ) + + +def _catalog_http_url(context: str, value: object) -> str: + url = _required_catalog_string(context, value) + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise PluginCatalogError(f"{context} {url!r} is invalid; expected an absolute HTTP(S) URL") + return url + + +def _format_catalog_keys(keys: set[str]) -> str: + return ", ".join(sorted(keys)) + + +def _format_catalog_choices(choices: set[str]) -> str: + return ", ".join(repr(choice) for choice in sorted(choices)) diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py index 5ef0dcbeb..b1b3bca9b 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py @@ -19,11 +19,11 @@ PLUGIN_TAP_CACHE_DIR_NAME, PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, PLUGIN_TAPS_FILE_NAME, - SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS, PluginCatalog, PluginCatalogError, PluginTapConfig, PluginTapRegistry, + validate_plugin_catalog_payload, ) from data_designer.cli.repositories.base import ConfigRepository from data_designer.config.utils.io_helpers import load_config_file, save_config_file @@ -167,7 +167,7 @@ def _load_cached_catalog(self, tap: PluginTapConfig, *, require_fresh: bool) -> except Exception: return None - def _save_catalog_cache(self, tap: PluginTapConfig, catalog_payload: dict) -> None: + def _save_catalog_cache(self, tap: PluginTapConfig, catalog_payload: dict[str, object]) -> None: self.cache_dir.mkdir(parents=True, exist_ok=True) cache_payload = { "tap_alias": tap.alias, @@ -189,17 +189,11 @@ def _fetch_catalog_payload(location: str) -> dict: @staticmethod def _validate_catalog(payload: dict, *, source: str) -> PluginCatalog: + validate_plugin_catalog_payload(payload, source=source) try: catalog = PluginCatalog.model_validate(payload) except ValidationError as e: raise PluginCatalogError(f"Invalid plugin catalog at {source!r}: {e}") from e - - if catalog.schema_version not in SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS: - supported = ", ".join(str(version) for version in sorted(SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS)) - raise PluginCatalogError( - f"Unsupported plugin catalog schema_version {catalog.schema_version!r} at {source!r}. " - f"Supported versions: {supported}." - ) return catalog @staticmethod diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index 7530365cd..66773c585 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -244,9 +244,6 @@ def _entry_search_text(entry: PluginCatalogEntry) -> str: entry.source.package if entry.source is not None and entry.source.package else "", entry.source.url if entry.source is not None and entry.source.url else "", entry.docs.url if entry.docs is not None and entry.docs.url else "", - entry.release_notes_url or "", - *_stringify_extra_values(entry.tags), - *_stringify_extra_values(entry.maintainers), ] return " ".join(values).lower() @@ -258,10 +255,6 @@ def _entry_version_sort_key(entry: PluginCatalogEntry) -> Version: return Version("0") -def _stringify_extra_values(values: Iterable[str]) -> list[str]: - return [str(value) for value in values] - - def _major_minor(version: str) -> str: parts = version.split(".") if len(parts) < 2: diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 6f3741c8f..2fed23db5 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -93,14 +93,11 @@ def _install_args_for_entry(entry: PluginCatalogEntry, tap: PluginTapConfig) -> target = _pypi_target(entry, source) return [target], target if source_type == "git": - target = _git_target(source) + target = _git_target(entry, source) return [target], target if source_type == "path": args = _path_args(entry, source, tap) return args, " ".join(args) - if source_type == "url": - target = _required(source.url, "url", source_type) - return [target], target raise ValueError(f"Plugin {entry.name!r} declares unsupported install source type {source.type!r}") @@ -112,25 +109,15 @@ def _pypi_target(entry: PluginCatalogEntry, source: PluginSourceInfo) -> str: return package_name -def _git_target(source: PluginSourceInfo) -> str: +def _git_target(entry: PluginCatalogEntry, source: PluginSourceInfo) -> str: url = _required(source.url, "url", "git") - target = url if url.startswith("git+") else f"git+{url}" - if source.ref: - target = f"{target}@{source.ref}" - - fragments = [] - if source.subdirectory: - fragments.append(f"subdirectory={source.subdirectory}") - if fragments: - target = f"{target}#{'&'.join(fragments)}" - return target + ref = _required(source.ref, "ref", "git") + subdirectory = _required(source.subdirectory, "subdirectory", "git") + return f"{entry.package.name} @ git+{url}@{ref}#subdirectory={subdirectory}" def _path_args(entry: PluginCatalogEntry, source: PluginSourceInfo, tap: PluginTapConfig) -> list[str]: - path = source.path or entry.package.path - if path is None: - raise ValueError(f"Plugin {entry.name!r} declares a path source without a path") - + path = _required(source.path, "path", "path") normalized_path = str(_resolve_path_source(path, tap)) if source.editable: return ["-e", normalized_path] diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py index 774bf6a6a..444a6cdf8 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py @@ -67,47 +67,156 @@ def test_load_catalog_rejects_unsupported_schema_version(tmp_path: Path) -> None repository = PluginTapRepository(tmp_path) repository.add_tap("local", str(catalog_path)) - with pytest.raises(PluginCatalogError, match="Unsupported plugin catalog schema_version"): + with pytest.raises(PluginCatalogError, match="unsupported catalog schema_version"): repository.load_catalog("local", refresh=True) -def _write_catalog(tmp_path: Path, *, schema_version: int = 2, plugin_name: str = "text-transform") -> Path: +def test_load_catalog_accepts_schema_v2_source_union(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + plugins=[ + _plugin_entry( + "pypi-plugin", + package_name="data-designer-pypi-plugin", + source={"type": "pypi", "package": "data-designer-pypi-plugin"}, + ), + _plugin_entry( + "git-plugin", + package_name="data-designer-git-plugin", + source={ + "type": "git", + "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", + "ref": "data-designer-git-plugin/v0.1.0", + "subdirectory": "plugins/data-designer-git-plugin", + }, + ), + _plugin_entry( + "path-plugin", + package_name="data-designer-path-plugin", + source={ + "type": "path", + "path": "plugins/data-designer-path-plugin", + "editable": True, + }, + ), + ], + ) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + catalog = repository.load_catalog("local", refresh=True) + + assert [entry.name for entry in catalog.plugins] == ["pypi-plugin", "git-plugin", "path-plugin"] + + +def test_load_catalog_rejects_invalid_schema_v2_source(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + plugins=[ + _plugin_entry( + "invalid-git-source", + package_name="data-designer-invalid-git-source", + source={ + "type": "git", + "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", + }, + ) + ], + ) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="invalid 'git' source fields"): + repository.load_catalog("local", refresh=True) + + +def test_load_catalog_rejects_unexpected_schema_v2_fields(tmp_path: Path) -> None: + plugin = _plugin_entry("text-transform") + plugin["tags"] = ["extra"] + catalog_path = _write_catalog(tmp_path, plugins=[plugin]) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="catalog plugins\\[0\\] has invalid fields"): + repository.load_catalog("local", refresh=True) + + +def test_load_catalog_rejects_duplicate_runtime_plugin_names(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + plugins=[ + _plugin_entry("duplicate", package_name="data-designer-one"), + _plugin_entry("duplicate", package_name="data-designer-two"), + ], + ) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="duplicate runtime plugin name"): + repository.load_catalog("local", refresh=True) + + +def _write_catalog( + tmp_path: Path, + *, + schema_version: int = 2, + plugin_name: str = "text-transform", + plugins: list[dict] | None = None, +) -> Path: catalog_dir = tmp_path / "catalog" catalog_dir.mkdir() catalog_path = catalog_dir / "plugins.json" - catalog_path.write_text(json.dumps(_catalog_payload(schema_version=schema_version, plugin_name=plugin_name))) + catalog_path.write_text( + json.dumps(_catalog_payload(schema_version=schema_version, plugin_name=plugin_name, plugins=plugins)) + ) return catalog_path -def _catalog_payload(*, schema_version: int = 2, plugin_name: str = "text-transform") -> dict: +def _catalog_payload( + *, + schema_version: int = 2, + plugin_name: str = "text-transform", + plugins: list[dict] | None = None, +) -> dict: return { "schema_version": schema_version, - "plugins": [ - { - "name": plugin_name, - "plugin_type": "processor", - "description": "Transform text records", - "package": { - "name": "data-designer-text-transform", - "version": "0.1.0", - "path": "plugins/data-designer-text-transform", - }, - "entry_point": { - "group": "data_designer.plugins", - "name": plugin_name, - "value": "data_designer_text_transform.plugin:plugin", - }, - "compatibility": { - "python": {"specifier": ">=3.10"}, - "data_designer": {"specifier": ">=0.5.7"}, - }, - "source": { - "type": "pypi", - "package": "data-designer-text-transform", - }, - "docs": { - "url": "https://example.com/text-transform", - }, + "plugins": plugins if plugins is not None else [_plugin_entry(plugin_name)], + } + + +def _plugin_entry( + plugin_name: str, + *, + package_name: str = "data-designer-text-transform", + source: dict | None = None, +) -> dict: + return { + "name": plugin_name, + "plugin_type": "processor", + "description": "Transform text records", + "package": { + "name": package_name, + "version": "0.1.0", + "path": f"plugins/{package_name}", + }, + "entry_point": { + "group": "data_designer.plugins", + "name": plugin_name, + "value": f"{package_name.replace('-', '_')}.plugin:plugin", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": None, }, - ], + }, + "source": source or { + "type": "pypi", + "package": package_name, + }, + "docs": { + "url": f"https://docs.example.test/plugins/{package_name}/", + }, } diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index 6f2c50257..fb702a410 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -7,6 +7,8 @@ from pathlib import Path from unittest.mock import Mock, patch +import pytest + from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository from data_designer.cli.services.plugin_catalog_service import PluginCatalogService from data_designer.plugins.plugin import PluginType @@ -23,30 +25,25 @@ def test_list_entries_filters_incompatible_plugins_by_default(tmp_path: Path) -> "compatible-plugin", "shared-column", "shared-processor", - "versioned-plugin", - "versioned-plugin", ] assert [entry.name for entry in all_entries] == [ "compatible-plugin", "future-plugin", "shared-column", "shared-processor", - "versioned-plugin", - "versioned-plugin", - "versioned-plugin", ] -def test_search_entries_matches_name_type_package_and_tags(tmp_path: Path) -> None: +def test_search_entries_matches_name_type_package_and_docs(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") name_matches = service.search_entries("compatible", "local") - tag_matches = service.search_entries("github", "local") + package_matches = service.search_entries("shared-package", "local") type_matches = service.search_entries("seed-reader", "local") assert [entry.name for entry in name_matches] == ["compatible-plugin"] - assert [entry.name for entry in tag_matches] == ["compatible-plugin"] + assert [entry.name for entry in package_matches] == ["shared-column", "shared-processor"] assert [entry.name for entry in type_matches] == ["compatible-plugin"] @@ -76,15 +73,12 @@ def test_evaluate_compatibility_accepts_local_dev_version_above_lower_bound(tmp_ assert result.reasons == [] -def test_get_entry_returns_newest_compatible_version(tmp_path: Path) -> None: +def test_get_entry_rejects_incompatible_plugin_when_requested(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - entry = service.get_entry("versioned-plugin", "local", include_incompatible=False) - newest_entry = service.get_entry("versioned-plugin", "local", include_incompatible=True) - - assert entry.package.version == "0.2.0" - assert newest_entry.package.version == "99.0.0" + with pytest.raises(ValueError, match="no compatible version"): + service.get_entry("future-plugin", "local", include_incompatible=False) def test_group_entries_by_package_groups_multi_plugin_packages(tmp_path: Path) -> None: @@ -136,52 +130,24 @@ def _catalog_payload() -> dict: plugin_type="seed-reader", package_name="data-designer-compatible-plugin", data_designer_specifier=">=0.5.7", - tags=["github", "repository"], ), _entry( name="future-plugin", plugin_type="processor", package_name="data-designer-future-plugin", data_designer_specifier=">=99.0", - tags=["future"], - ), - _entry( - name="versioned-plugin", - plugin_type="processor", - package_name="data-designer-versioned-plugin", - data_designer_specifier=">=0.5.7", - package_version="0.1.0", - tags=["versioned"], - ), - _entry( - name="versioned-plugin", - plugin_type="processor", - package_name="data-designer-versioned-plugin", - data_designer_specifier=">=0.5.7", - package_version="0.2.0", - tags=["versioned"], - ), - _entry( - name="versioned-plugin", - plugin_type="processor", - package_name="data-designer-versioned-plugin", - data_designer_specifier=">=99.0", - package_version="99.0.0", - tags=["versioned"], ), _entry( name="shared-column", plugin_type="column-generator", package_name="data-designer-shared-package", data_designer_specifier=">=0.5.7", - tags=["shared"], ), _entry( name="shared-processor", plugin_type="processor", package_name="data-designer-shared-package", data_designer_specifier=">=0.5.7", - tags=["shared"], ), ], } @@ -193,7 +159,6 @@ def _entry( plugin_type: str, package_name: str, data_designer_specifier: str, - tags: list[str], package_version: str = "0.1.0", ) -> dict: return { @@ -203,6 +168,7 @@ def _entry( "package": { "name": package_name, "version": package_version, + "path": f"plugins/{package_name}", }, "entry_point": { "group": "data_designer.plugins", @@ -211,11 +177,17 @@ def _entry( }, "compatibility": { "python": {"specifier": ">=3.10"}, - "data_designer": {"specifier": data_designer_specifier}, + "data_designer": { + "requirement": f"data-designer{data_designer_specifier}", + "specifier": data_designer_specifier, + "marker": None, + }, }, "source": { "type": "pypi", "package": package_name, }, - "tags": tags, + "docs": { + "url": f"https://docs.example.test/plugins/{package_name}/", + }, } diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index b75e1a557..15abdaf86 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -45,7 +45,7 @@ def test_build_git_install_plan_includes_ref_and_subdirectory() -> None: plan = service.build_install_plan(entry, tap, manager="pip") assert plan.command[-1] == ( - "git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git" + "data-designer-text-transform @ git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git" "@data-designer-text-transform/v0.1.0" "#subdirectory=plugins/data-designer-text-transform" ) @@ -62,8 +62,28 @@ def test_build_uv_install_plan_targets_current_python(mock_which: Mock) -> None: assert plan.command[:5] == ["uv", "pip", "install", "--python", sys.executable] +def test_build_git_install_plan_requires_ref_and_subdirectory() -> None: + entry = _entry( + source={ + "type": "git", + "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", + } + ) + tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + with pytest.raises(ValueError, match="requires 'ref'"): + service.build_install_plan(entry, tap, manager="pip") + + def test_build_path_install_plan_resolves_relative_path_from_local_tap_root(tmp_path: Path) -> None: - entry = _entry(source={"type": "path", "editable": True}) + entry = _entry( + source={ + "type": "path", + "path": "plugins/data-designer-text-transform", + "editable": True, + } + ) tap = PluginTapConfig(alias="local", url=str(tmp_path / "catalog" / "plugins.json")) service = PluginInstallService() @@ -128,8 +148,15 @@ def _entry(source: dict | None) -> PluginCatalogEntry: }, "compatibility": { "python": {"specifier": ">=3.10"}, - "data_designer": {"specifier": ">=0.5.7"}, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": None, + }, }, "source": source, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-text-transform/", + }, } return PluginCatalogEntry.model_validate(payload) From 47bed27e887577a254af8987d45899123beb3757 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 20:54:57 +0000 Subject: [PATCH 05/34] fix(cli): address plugin review feedback - Invalidate import caches before post-install entry point verification - Make tap aliases case-insensitive and cache catalogs by alias plus URL - Prefer compatible catalog entries before falling back to forced installs - Clarify unused --tap behavior and list installed entry points without imports - Add direct controller coverage and update CLI plugin documentation Refs #617 --- .../src/data_designer/cli/README.md | 56 +++- .../src/data_designer/cli/commands/plugins.py | 29 +- .../controllers/plugin_catalog_controller.py | 26 +- .../src/data_designer/cli/main.py | 2 +- .../src/data_designer/cli/plugin_catalog.py | 13 +- .../cli/repositories/plugin_tap_repository.py | 52 +++- .../cli/services/plugin_catalog_service.py | 40 +-- .../cli/services/plugin_install_service.py | 2 + .../cli/commands/test_plugins_command.py | 36 +++ .../test_plugin_catalog_controller.py | 267 ++++++++++++++++++ .../test_plugin_tap_repository.py | 44 ++- .../services/test_plugin_catalog_service.py | 89 +++++- .../services/test_plugin_install_service.py | 7 +- uv.lock | 6 +- 14 files changed, 591 insertions(+), 78 deletions(-) create mode 100644 packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index f15b752e9..338912ea4 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -1,12 +1,14 @@ # 🎨 NeMo Data Designer CLI -This directory contains the Command-Line Interface (CLI) for configuring model providers and model configurations used in Data Designer. +This directory contains the Command-Line Interface (CLI) for configuring model providers, model configurations, managed assets, and plugin tap catalogs used in Data Designer. ## Overview The CLI provides an interactive interface for managing: - **Model Providers**: LLM API endpoints (NVIDIA, OpenAI, Anthropic, custom providers) - **Model Configs**: Specific model configurations with inference parameters +- **Plugin Taps**: Catalog aliases for discovering Data Designer plugin packages +- **Plugin Installs**: Safe install-plan rendering, package-manager execution, and entry point verification Configuration files are stored in `~/.data-designer/` by default and can be referenced by Data Designer workflows. @@ -17,7 +19,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis ``` ┌─────────────────────────────────────────────────────────────┐ │ Commands │ -│ Entry points for CLI commands (list, providers, models) │ +│ Entry points for CLI commands (list, providers, plugins) │ └─────────────────────────────────────────────────────────────┘ │ ▼ @@ -52,6 +54,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `list.py`: List current configurations - `models.py`: Configure models - `providers.py`: Configure providers + - `plugins.py`: Discover and install plugins from tap catalogs - `reset.py`: Reset/delete configurations #### 2. **Controllers** (`controllers/`) @@ -64,6 +67,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - **Files**: - `model_controller.py`: Orchestrates model configuration workflows - `provider_controller.py`: Orchestrates provider configuration workflows + - `plugin_catalog_controller.py`: Orchestrates plugin catalog, tap, and install workflows **Key Features**: - **Associated Resource Management**: When deleting a provider, the controller checks for associated models and prompts the user to delete them together @@ -79,6 +83,8 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - **Files**: - `model_service.py`: Model configuration business logic - `provider_service.py`: Provider business logic + - `plugin_catalog_service.py`: Plugin catalog discovery, search, compatibility checks, and installed entry point listing + - `plugin_install_service.py`: Plugin install-plan resolution, package-manager execution, and runtime verification **Key Methods**: - `list_all()`: Get all configured items @@ -101,6 +107,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `base.py`: Abstract base repository with common operations - `model_repository.py`: Model configuration persistence - `provider_repository.py`: Provider persistence + - `plugin_tap_repository.py`: Plugin tap aliases, catalog fetching, and URL-keyed catalog cache **Base Repository Pattern**: ```python @@ -152,7 +159,7 @@ class ConfigRepository(ABC, Generic[T]): ## Configuration Files -The CLI manages two YAML configuration files: +The CLI manages YAML configuration files and plugin catalog caches under `~/.data-designer/`: ### `~/.data-designer/model_providers.yaml` @@ -206,6 +213,22 @@ model_configs: max_parallel_requests: 4 ``` +### `~/.data-designer/plugin_taps.yaml` + +Stores user-added plugin tap aliases. The built-in NVIDIA tap is always available and is not written to this file. Set `DATA_DESIGNER_DEFAULT_PLUGIN_TAP_URL` to repoint the built-in tap for QA or staging. + +```yaml +taps: + - alias: research + url: https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json + trusted: false + cache_ttl_seconds: 86400 +``` + +### `~/.data-designer/plugin-tap-cache/` + +Stores fetched plugin catalog payloads as JSON cache files keyed by tap alias and URL hash. This prevents a re-pointed alias from serving stale catalog data from a previous URL. + ## Usage Examples ### Configure Providers @@ -248,3 +271,30 @@ data-designer config list # Delete configuration files (with confirmation) data-designer config reset ``` + +### Discover and Install Plugins + +```bash +# List compatible plugins from the default NVIDIA tap +data-designer plugins list + +# Search a specific tap catalog +data-designer plugins --tap research search transform + +# Show metadata, compatibility, docs, and exact install command +data-designer plugins info text-transform + +# Install from a trusted or user-added tap and verify entry point discovery +data-designer plugins install text-transform --yes + +# Preview the install command without mutating the environment +data-designer plugins install text-transform --dry-run + +# Add and manage tap aliases +data-designer plugins taps add research https://github.com/acme/dd-plugins +data-designer plugins taps list +data-designer plugins taps remove research + +# List installed plugin entry points without importing plugin modules +data-designer plugins installed +``` diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugins.py index de2b1afff..e58e862f2 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugins.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugins.py @@ -7,6 +7,7 @@ import typer from data_designer.cli.controllers.plugin_catalog_controller import PluginCatalogController +from data_designer.cli.ui import print_info from data_designer.config.utils.constants import DATA_DESIGNER_HOME @@ -39,7 +40,7 @@ def list_command( def search_command( ctx: typer.Context, - query: str = typer.Argument(help="Keyword, plugin type, package name, source, maintainer, or tag to search for."), + query: str = typer.Argument(help="Keyword, plugin type, package name, source, docs URL, or entry point to search for."), tap: str | None = typer.Option( None, "--tap", @@ -122,7 +123,7 @@ def install_command( force: bool = typer.Option( False, "--force", - help="Allow installation even when catalog compatibility checks fail.", + help="Allow installation when only incompatible catalog entries are available.", ), ) -> None: """Install one Data Designer plugin package, then verify runtime discovery.""" @@ -138,19 +139,22 @@ def install_command( ) -def installed_command() -> None: - """List installed Data Designer plugins discovered from runtime entry points.""" +def installed_command(ctx: typer.Context) -> None: + """List installed Data Designer plugin entry points.""" + _warn_if_parent_tap_unused(ctx, "installed plugins are discovered from the current Python environment") controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_installed() -def taps_list_command() -> None: +def taps_list_command(ctx: typer.Context) -> None: """List configured plugin taps.""" + _warn_if_parent_tap_unused(ctx, "tap management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_taps_list() def taps_add_command( + ctx: typer.Context, alias: str = typer.Argument(help="Local alias for the plugin tap."), url: str = typer.Argument(help="Tap repository URL, catalog URL, local catalog file, or local tap directory."), trusted: bool = typer.Option( @@ -166,6 +170,7 @@ def taps_add_command( ), ) -> None: """Add a plugin tap alias.""" + _warn_if_parent_tap_unused(ctx, "tap management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_taps_add( alias=alias, @@ -176,9 +181,11 @@ def taps_add_command( def taps_remove_command( + ctx: typer.Context, alias: str = typer.Argument(help="Plugin tap alias to remove."), ) -> None: """Remove a plugin tap alias.""" + _warn_if_parent_tap_unused(ctx, "tap management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_taps_remove(alias=alias) @@ -187,6 +194,12 @@ def _resolve_tap_alias(ctx: typer.Context, tap_alias: str | None) -> str | None: if tap_alias is not None: return tap_alias + return _parent_tap_alias(ctx) + + +def _parent_tap_alias(ctx: typer.Context) -> str | None: + """Return --tap from the plugins parent command when present.""" + parent = ctx.parent while parent is not None: candidate = parent.params.get("tap") if parent.params else None @@ -194,3 +207,9 @@ def _resolve_tap_alias(ctx: typer.Context, tap_alias: str | None) -> str | None: return candidate parent = parent.parent return None + + +def _warn_if_parent_tap_unused(ctx: typer.Context, reason: str) -> None: + tap_alias = _parent_tap_alias(ctx) + if tap_alias is not None: + print_info(f"Ignoring --tap {tap_alias!r}; {reason}.") diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 2bafd6102..8a8357658 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -7,10 +7,12 @@ from pathlib import Path import typer +from pydantic import ValidationError from rich.table import Table from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_TAP_ALIAS, + PLUGIN_TAP_ALIAS_PATTERN, CompatibilityResult, InstalledPluginInfo, PluginCatalogEntry, @@ -33,7 +35,11 @@ class PluginCatalogController: - """Controller for plugin catalog, tap, and install workflows.""" + """Controller for plugin catalog, tap, and install workflows. + + Catalog browsing and environment mutation intentionally use separate services so + read-only tap operations stay decoupled from package-manager execution. + """ def __init__(self, config_dir: Path) -> None: self.config_dir = config_dir @@ -182,7 +188,7 @@ def run_install( ) def run_installed(self) -> None: - """List plugins currently discoverable through runtime entry points.""" + """List installed plugin entry points without importing plugin modules.""" print_header("Installed Data Designer Plugins") installed_plugins = self.catalog_service.list_installed_plugins() if not installed_plugins: @@ -225,6 +231,12 @@ def run_taps_add( trusted=trusted, cache_ttl_seconds=cache_ttl_seconds, ) + except ValidationError as e: + if any(tuple(error["loc"]) == ("alias",) for error in e.errors()): + print_error(f"Invalid tap alias {alias!r}: must match `{PLUGIN_TAP_ALIAS_PATTERN}`") + else: + print_error(f"Invalid plugin tap configuration: {e}") + raise typer.Exit(code=1) except Exception as e: print_error(f"Failed to add plugin tap: {e}") raise typer.Exit(code=1) @@ -288,7 +300,7 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: table = Table(title="Catalog Plugins", border_style=NordColor.NORD8.value) table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) table.add_column("Type", style=NordColor.NORD9.value, no_wrap=True) - table.add_column("Package", style=NordColor.NORD4.value) + table.add_column("Package", style=NordColor.NORD4.value, no_wrap=True) table.add_column("Version", style=NordColor.NORD15.value, no_wrap=True) table.add_column("Compatible", style=NordColor.NORD13.value, no_wrap=True) table.add_column("Docs", style=NordColor.NORD7.value) @@ -310,16 +322,12 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: def _display_installed_plugins(installed_plugins: list[InstalledPluginInfo]) -> None: table = Table(title="Installed Plugins", border_style=NordColor.NORD8.value) table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) - table.add_column("Type", style=NordColor.NORD9.value, no_wrap=True) - table.add_column("Config", style=NordColor.NORD4.value) - table.add_column("Implementation", style=NordColor.NORD7.value) + table.add_column("Entry Point", style=NordColor.NORD4.value) for plugin in installed_plugins: table.add_row( plugin.name, - plugin.plugin_type.value, - plugin.config_qualified_name, - plugin.impl_qualified_name, + plugin.entry_point_value, ) console.print(table) diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index f85e78df7..993d34a07 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -170,7 +170,7 @@ def _is_version_request(args: list[str]) -> bool: "installed": { "module": f"{_CMD}.plugins", "attr": "installed_command", - "help": "List installed runtime plugins", + "help": "List installed plugin entry points", }, } ), diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index f3b5564f6..915ec317b 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import re from dataclasses import dataclass from urllib.parse import urlparse @@ -17,6 +18,7 @@ DEFAULT_PLUGIN_TAP_ALIAS = "nvidia" DEFAULT_PLUGIN_TAP_URL = "https://raw.githubusercontent.com/NVIDIA-NeMo/DataDesignerPlugins/main/catalog/plugins.json" +DEFAULT_PLUGIN_TAP_URL_ENV_VAR = "DATA_DESIGNER_DEFAULT_PLUGIN_TAP_URL" PLUGIN_TAPS_FILE_NAME = "plugin_taps.yaml" PLUGIN_TAP_CACHE_DIR_NAME = "plugin-tap-cache" PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS = 24 * 60 * 60 @@ -175,12 +177,15 @@ class InstallPlan: @dataclass(frozen=True) class InstalledPluginInfo: - """Runtime plugin discovered from installed entry points.""" + """Installed plugin entry point discovered without importing plugin code.""" name: str - plugin_type: PluginType - config_qualified_name: str - impl_qualified_name: str + entry_point_value: str + + +def get_default_plugin_tap_url() -> str: + """Return the built-in plugin tap URL, honoring a local override for QA/staging.""" + return os.getenv(DEFAULT_PLUGIN_TAP_URL_ENV_VAR, DEFAULT_PLUGIN_TAP_URL) def validate_plugin_catalog_payload(payload: object, *, source: str) -> None: diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py index b1b3bca9b..2e896a1bb 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py @@ -3,6 +3,7 @@ from __future__ import annotations +import hashlib import json from datetime import datetime, timezone from pathlib import Path @@ -14,7 +15,6 @@ from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_TAP_ALIAS, - DEFAULT_PLUGIN_TAP_URL, MAX_PLUGIN_CATALOG_SIZE_BYTES, PLUGIN_TAP_CACHE_DIR_NAME, PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, @@ -23,6 +23,7 @@ PluginCatalogError, PluginTapConfig, PluginTapRegistry, + get_default_plugin_tap_url, validate_plugin_catalog_payload, ) from data_designer.cli.repositories.base import ConfigRepository @@ -63,13 +64,13 @@ def list_taps(self) -> list[PluginTapConfig]: taps = [self.default_tap()] registry = self.load() if registry is not None: - taps.extend(sorted(registry.taps, key=lambda tap: tap.alias)) + taps.extend(sorted(registry.taps, key=lambda tap: tap.alias.casefold())) return taps def get_tap(self, alias: str | None = None) -> PluginTapConfig | None: """Return a tap by alias, defaulting to the built-in NVIDIA tap.""" resolved_alias = alias or DEFAULT_PLUGIN_TAP_ALIAS - return next((tap for tap in self.list_taps() if tap.alias == resolved_alias), None) + return next((tap for tap in self.list_taps() if _same_alias(tap.alias, resolved_alias)), None) def add_tap( self, @@ -95,7 +96,7 @@ def add_tap( ) registry = self.load() or PluginTapRegistry() registry.taps.append(tap) - registry.taps = sorted(registry.taps, key=lambda item: item.alias) + registry.taps = sorted(registry.taps, key=lambda item: item.alias.casefold()) self.save(registry) return tap @@ -105,21 +106,21 @@ def remove_tap(self, alias: str) -> None: Raises: ValueError: If the alias is reserved or does not exist. """ - if alias == DEFAULT_PLUGIN_TAP_ALIAS: + if _same_alias(alias, DEFAULT_PLUGIN_TAP_ALIAS): raise ValueError(f"Cannot remove the built-in {DEFAULT_PLUGIN_TAP_ALIAS!r} plugin tap") registry = self.load() - if registry is None or not any(tap.alias == alias for tap in registry.taps): + matching_tap = next((tap for tap in registry.taps if _same_alias(tap.alias, alias)), None) if registry else None + if registry is None or matching_tap is None: raise ValueError(f"Plugin tap alias {alias!r} not found") - registry.taps = [tap for tap in registry.taps if tap.alias != alias] + registry.taps = [tap for tap in registry.taps if not _same_alias(tap.alias, alias)] if registry.taps: self.save(registry) else: self.delete() - cache_file = self._cache_file(alias) - cache_file.unlink(missing_ok=True) + self._remove_cache_files(matching_tap) def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> PluginCatalog: """Load a tap catalog from cache or source.""" @@ -146,7 +147,7 @@ def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> Pl return catalog def _load_cached_catalog(self, tap: PluginTapConfig, *, require_fresh: bool) -> PluginCatalog | None: - cache_file = self._cache_file(tap.alias) + cache_file = self._cache_file(tap) if not cache_file.exists(): return None @@ -175,11 +176,30 @@ def _save_catalog_cache(self, tap: PluginTapConfig, catalog_payload: dict[str, o "fetched_at": datetime.now(timezone.utc).isoformat(), "catalog": catalog_payload, } - with open(self._cache_file(tap.alias), "w") as f: + with open(self._cache_file(tap), "w") as f: json.dump(cache_payload, f, indent=2, sort_keys=True) - def _cache_file(self, alias: str) -> Path: - return self.cache_dir / f"{alias}.json" + def _cache_file(self, tap: PluginTapConfig) -> Path: + url_hash = hashlib.sha256(tap.url.encode("utf-8")).hexdigest()[:12] + return self.cache_dir / f"{tap.alias}-{url_hash}.json" + + def _remove_cache_files(self, tap: PluginTapConfig) -> None: + if not self.cache_dir.exists(): + return + + self._cache_file(tap).unlink(missing_ok=True) + legacy_cache_file = self.cache_dir / f"{tap.alias}.json" + legacy_cache_file.unlink(missing_ok=True) + + for cache_file in self.cache_dir.glob("*.json"): + try: + with open(cache_file) as f: + cache_payload = json.load(f) + except Exception: + continue + cached_alias = cache_payload.get("tap_alias") + if isinstance(cached_alias, str) and _same_alias(cached_alias, tap.alias): + cache_file.unlink(missing_ok=True) @staticmethod def _fetch_catalog_payload(location: str) -> dict: @@ -201,7 +221,7 @@ def default_tap() -> PluginTapConfig: """Return the built-in NVIDIA plugin tap configuration.""" return PluginTapConfig( alias=DEFAULT_PLUGIN_TAP_ALIAS, - url=DEFAULT_PLUGIN_TAP_URL, + url=get_default_plugin_tap_url(), trusted=True, cache_ttl_seconds=PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, ) @@ -218,6 +238,10 @@ def normalize_tap_location(location: str) -> str: return str((path / "catalog" / "plugins.json").resolve(strict=False)) +def _same_alias(left: str, right: str) -> bool: + return left.casefold() == right.casefold() + + def _normalize_tap_url(url: str) -> str: parsed = urlparse(url) hostname = parsed.hostname or "" diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index 66773c585..b7aa1156e 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -10,6 +10,7 @@ from packaging.markers import InvalidMarker, Marker from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.utils import canonicalize_name from packaging.version import InvalidVersion, Version from data_designer.cli.plugin_catalog import ( @@ -21,8 +22,6 @@ PluginTapConfig, ) from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository -from data_designer.plugins.plugin import PluginType -from data_designer.plugins.registry import PluginRegistry class PluginCatalogService: @@ -48,7 +47,7 @@ def list_entries( ) -> list[PluginCatalogEntry]: """List catalog entries for a tap, filtering incompatible entries by default.""" catalog = self.repository.load_catalog(tap_alias, refresh=refresh) - entries = sorted(catalog.plugins, key=lambda entry: (entry.name, entry.package.version or "")) + entries = sorted(catalog.plugins, key=lambda entry: (entry.name, _entry_version_sort_key(entry))) if include_incompatible: return entries return [entry for entry in entries if self.evaluate_compatibility(entry).is_compatible] @@ -87,16 +86,14 @@ def get_entry( """Return the newest catalog entry by plugin name.""" entries = self.list_entries(tap_alias, refresh=refresh, include_incompatible=True) matches = [entry for entry in entries if entry.name == name] - matched_incompatible = False - if matches and not include_incompatible: - compatible_matches = [entry for entry in matches if self.evaluate_compatibility(entry).is_compatible] - matched_incompatible = bool(matches) and not compatible_matches - matches = compatible_matches - if matches: + compatible_matches = [entry for entry in matches if self.evaluate_compatibility(entry).is_compatible] + if compatible_matches: + return max(compatible_matches, key=_entry_version_sort_key) + if matches and include_incompatible: return max(matches, key=_entry_version_sort_key) resolved_alias = tap_alias or DEFAULT_PLUGIN_TAP_ALIAS - if matched_incompatible: + if matches: raise ValueError( f"Plugin {name!r} was found in tap {resolved_alias!r}, but no compatible version is available" ) @@ -107,7 +104,7 @@ def group_entries_by_package(entries: Iterable[PluginCatalogEntry]) -> dict[str, """Group catalog entries by installable package name.""" grouped_entries: dict[str, list[PluginCatalogEntry]] = defaultdict(list) for entry in entries: - grouped_entries[entry.package.name].append(entry) + grouped_entries[canonicalize_name(entry.package.name)].append(entry) return { package_name: sorted(items, key=lambda item: item.name) for package_name, items in grouped_entries.items() } @@ -169,20 +166,13 @@ def remove_tap(self, alias: str) -> None: self.repository.remove_tap(alias) def list_installed_plugins(self) -> list[InstalledPluginInfo]: - """List runtime plugins currently discoverable through entry points.""" - registry = PluginRegistry() - installed_plugins = [] - for plugin_type in PluginType: - for plugin in registry.get_plugins(plugin_type): - installed_plugins.append( - InstalledPluginInfo( - name=plugin.name, - plugin_type=plugin.plugin_type, - config_qualified_name=plugin.config_qualified_name, - impl_qualified_name=plugin.impl_qualified_name, - ) - ) - return sorted(installed_plugins, key=lambda plugin: (plugin.plugin_type.value, plugin.name)) + """List installed Data Designer plugin entry points without importing plugin modules.""" + entry_points = importlib.metadata.entry_points(group="data_designer.plugins") + installed_plugins = [ + InstalledPluginInfo(name=entry_point.name, entry_point_value=entry_point.value) + for entry_point in entry_points + ] + return sorted(installed_plugins, key=lambda plugin: plugin.name) def _evaluate_target( self, diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 2fed23db5..22b39e02f 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -3,6 +3,7 @@ from __future__ import annotations +import importlib import shutil import subprocess import sys @@ -55,6 +56,7 @@ def install(self, plan: InstallPlan) -> None: def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: """Verify the plugin is discoverable by the runtime PluginRegistry.""" + importlib.invalidate_caches() PluginRegistry.reset() registry = PluginRegistry() return registry.plugin_exists(entry.name) diff --git a/packages/data-designer/tests/cli/commands/test_plugins_command.py b/packages/data-designer/tests/cli/commands/test_plugins_command.py index c895d360d..c74f0e9e2 100644 --- a/packages/data-designer/tests/cli/commands/test_plugins_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugins_command.py @@ -88,3 +88,39 @@ def test_plugins_taps_add_command_delegates_to_controller(mock_ctrl_cls: MagicMo trusted=True, cache_ttl_seconds=60, ) + + +@patch("data_designer.cli.commands.plugins.print_info") +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_installed_warns_when_parent_tap_is_unused( + mock_ctrl_cls: MagicMock, + mock_print_info: MagicMock, +) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke(app, ["plugins", "--tap", "research", "installed"]) + + assert result.exit_code == 0 + mock_print_info.assert_called_once_with( + "Ignoring --tap 'research'; installed plugins are discovered from the current Python environment." + ) + mock_ctrl.run_installed.assert_called_once_with() + + +@patch("data_designer.cli.commands.plugins.print_info") +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_taps_list_warns_when_parent_tap_is_unused( + mock_ctrl_cls: MagicMock, + mock_print_info: MagicMock, +) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke(app, ["plugins", "--tap", "research", "taps", "list"]) + + assert result.exit_code == 0 + mock_print_info.assert_called_once_with( + "Ignoring --tap 'research'; tap management commands operate on aliases directly." + ) + mock_ctrl.run_taps_list.assert_called_once_with() diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py new file mode 100644 index 000000000..2851be8d4 --- /dev/null +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -0,0 +1,267 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import typer + +from data_designer.cli.controllers.plugin_catalog_controller import PluginCatalogController +from data_designer.cli.plugin_catalog import ( + CompatibilityResult, + InstallPlan, + PluginCatalogEntry, + PluginTapConfig, +) + + +@pytest.fixture +def controller(tmp_path: Path) -> PluginCatalogController: + plugin_controller = PluginCatalogController(tmp_path) + plugin_controller.catalog_service = MagicMock() + plugin_controller.install_service = MagicMock() + return plugin_controller + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") +def test_run_install_dry_run_renders_plan_without_installing( + mock_print_info: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + tap = _tap(trusted=True) + plan = _plan(tap) + controller.catalog_service.get_tap.return_value = tap + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = plan + + controller.run_install("text-transform", tap_alias="local", dry_run=True) + + controller.catalog_service.get_entry.assert_called_once_with( + "text-transform", + "local", + refresh=False, + include_incompatible=False, + ) + controller.install_service.install.assert_not_called() + controller.install_service.verify_entry_point.assert_not_called() + mock_print_info.assert_any_call("Dry run complete; no changes made") + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +def test_run_install_blocks_incompatible_plugin_without_force( + mock_print_error: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + tap = _tap(trusted=True) + controller.catalog_service.get_tap.return_value = tap + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( + False, + ["Data Designer 0.5.7 does not satisfy >=99.0"], + ) + + with pytest.raises(typer.Exit) as exc_info: + controller.run_install("text-transform", tap_alias="local") + + assert exc_info.value.exit_code == 1 + controller.catalog_service.get_entry.assert_called_once_with( + "text-transform", + "local", + refresh=False, + include_incompatible=False, + ) + controller.install_service.build_install_plan.assert_not_called() + mock_print_error.assert_called_once_with("Plugin 'text-transform' is not compatible with this environment") + mock_console.print.assert_any_call(" - Data Designer 0.5.7 does not satisfy >=99.0") + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") +def test_run_install_force_allows_incompatible_entry_for_dry_run( + mock_print_info: MagicMock, + mock_print_error: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + tap = _tap(trusted=True) + controller.catalog_service.get_tap.return_value = tap + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( + False, + ["Data Designer 0.5.7 does not satisfy >=99.0"], + ) + controller.install_service.build_install_plan.return_value = _plan(tap) + + controller.run_install("text-transform", tap_alias="local", dry_run=True, force=True) + + controller.catalog_service.get_entry.assert_called_once_with( + "text-transform", + "local", + refresh=False, + include_incompatible=True, + ) + controller.install_service.build_install_plan.assert_called_once_with(entry, tap, manager="auto") + controller.install_service.install.assert_not_called() + mock_print_error.assert_not_called() + mock_print_info.assert_any_call("Dry run complete; no changes made") + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_install_warns_for_untrusted_tap( + mock_print_warning: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + tap = _tap(trusted=False) + controller.catalog_service.get_tap.return_value = tap + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = _plan(tap) + + controller.run_install("text-transform", tap_alias="local", dry_run=True) + + mock_print_warning.assert_called_once_with( + "This tap is not marked trusted. Plugin installation executes Python package code from the source above." + ) + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_success") +def test_run_install_reports_success_when_verification_finds_entry_point( + mock_print_success: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + tap = _tap(trusted=True) + plan = _plan(tap) + controller.catalog_service.get_tap.return_value = tap + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = plan + controller.install_service.verify_entry_point.return_value = True + + controller.run_install("text-transform", tap_alias="local", yes=True) + + controller.install_service.install.assert_called_once_with(plan) + controller.install_service.verify_entry_point.assert_called_once_with(entry) + mock_print_success.assert_called_once_with("Plugin 'text-transform' installed and discovered") + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_install_warns_when_verification_misses_entry_point( + mock_print_warning: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + tap = _tap(trusted=True) + plan = _plan(tap) + controller.catalog_service.get_tap.return_value = tap + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = plan + controller.install_service.verify_entry_point.return_value = False + + controller.run_install("text-transform", tap_alias="local", yes=True) + + controller.install_service.install.assert_called_once_with(plan) + controller.install_service.verify_entry_point.assert_called_once_with(entry) + mock_print_warning.assert_called_once_with( + "Plugin 'text-transform' was installed, but Data Designer did not discover its entry point. " + "Restart the shell or check the package entry point metadata." + ) + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +def test_run_taps_add_wraps_invalid_alias_validation_error( + mock_print_error: MagicMock, + tmp_path: Path, +) -> None: + plugin_controller = PluginCatalogController(tmp_path) + + with pytest.raises(typer.Exit) as exc_info: + plugin_controller.run_taps_add( + alias="foo/bar", + url="https://github.com/acme/dd-plugins", + trusted=False, + cache_ttl_seconds=60, + ) + + assert exc_info.value.exit_code == 1 + mock_print_error.assert_called_once_with("Invalid tap alias 'foo/bar': must match `^[A-Za-z0-9_.-]+$`") + + +def _tap(*, trusted: bool) -> PluginTapConfig: + return PluginTapConfig( + alias="local", + url="https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json", + trusted=trusted, + ) + + +def _plan(tap: PluginTapConfig) -> InstallPlan: + return InstallPlan( + plugin_name="text-transform", + package_name="data-designer-text-transform", + source_description="data-designer-text-transform==0.1.0", + command=["python", "-m", "pip", "install", "data-designer-text-transform==0.1.0"], + manager="pip", + tap_alias=tap.alias, + trusted_tap=tap.trusted, + ) + + +def _entry() -> PluginCatalogEntry: + return PluginCatalogEntry.model_validate( + { + "name": "text-transform", + "plugin_type": "processor", + "description": "Transform text records", + "package": { + "name": "data-designer-text-transform", + "version": "0.1.0", + "path": "plugins/data-designer-text-transform", + }, + "entry_point": { + "group": "data_designer.plugins", + "name": "text-transform", + "value": "data_designer_text_transform.plugin:plugin", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": None, + }, + }, + "source": { + "type": "pypi", + "package": "data-designer-text-transform", + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-text-transform/", + }, + } + ) diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py index 444a6cdf8..eeed579b0 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py @@ -8,7 +8,11 @@ import pytest -from data_designer.cli.plugin_catalog import DEFAULT_PLUGIN_TAP_ALIAS, PluginCatalogError +from data_designer.cli.plugin_catalog import ( + DEFAULT_PLUGIN_TAP_ALIAS, + DEFAULT_PLUGIN_TAP_URL_ENV_VAR, + PluginCatalogError, +) from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository, normalize_tap_location @@ -21,6 +25,15 @@ def test_repository_includes_default_nvidia_tap(tmp_path: Path) -> None: assert taps[0].trusted is True +def test_default_tap_honors_url_environment_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(DEFAULT_PLUGIN_TAP_URL_ENV_VAR, "https://example.test/catalog/plugins.json") + repository = PluginTapRepository(tmp_path) + + tap = repository.default_tap() + + assert tap.url == "https://example.test/catalog/plugins.json" + + def test_add_tap_normalizes_github_repository_url(tmp_path: Path) -> None: repository = PluginTapRepository(tmp_path) @@ -30,6 +43,22 @@ def test_add_tap_normalizes_github_repository_url(tmp_path: Path) -> None: assert repository.get_tap("research") == tap +def test_tap_aliases_are_case_insensitive(tmp_path: Path) -> None: + repository = PluginTapRepository(tmp_path) + + tap = repository.add_tap("Research", "https://github.com/acme/dd-plugins") + + assert repository.get_tap("research") == tap + with pytest.raises(ValueError, match="already exists"): + repository.add_tap("research", "https://github.com/acme/other-plugins") + with pytest.raises(ValueError, match="already exists"): + repository.add_tap("NVIDIA", "https://github.com/acme/nvidia-plugins") + + repository.remove_tap("research") + + assert repository.get_tap("Research") is None + + def test_normalize_local_tap_directory() -> None: normalized = normalize_tap_location("~/plugins") @@ -62,6 +91,19 @@ def test_load_catalog_with_zero_cache_ttl_refreshes_source(tmp_path: Path) -> No assert refreshed_catalog.plugins[0].name == "fresh-transform" +def test_load_catalog_cache_file_is_keyed_by_alias_and_url(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path) + repository = PluginTapRepository(tmp_path) + repository.add_tap("local", str(catalog_path)) + + repository.load_catalog("local") + + cache_files = list(repository.cache_dir.glob("*.json")) + assert len(cache_files) == 1 + assert cache_files[0].name.startswith("local-") + assert cache_files[0].name != "local.json" + + def test_load_catalog_rejects_unsupported_schema_version(tmp_path: Path) -> None: catalog_path = _write_catalog(tmp_path, schema_version=999) repository = PluginTapRepository(tmp_path) diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index fb702a410..98c876926 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -4,14 +4,15 @@ from __future__ import annotations import json +from importlib.metadata import EntryPoint from pathlib import Path from unittest.mock import Mock, patch import pytest +from data_designer.cli.plugin_catalog import PluginCatalog, PluginCatalogEntry from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository from data_designer.cli.services.plugin_catalog_service import PluginCatalogService -from data_designer.plugins.plugin import PluginType def test_list_entries_filters_incompatible_plugins_by_default(tmp_path: Path) -> None: @@ -81,6 +82,36 @@ def test_get_entry_rejects_incompatible_plugin_when_requested(tmp_path: Path) -> service.get_entry("future-plugin", "local", include_incompatible=False) +def test_get_entry_prefers_newest_compatible_match_when_include_incompatible() -> None: + repository = Mock(spec=PluginTapRepository) + repository.load_catalog.return_value = PluginCatalog.model_validate( + { + "schema_version": 2, + "plugins": [ + _entry( + name="versioned-plugin", + plugin_type="processor", + package_name="data-designer-versioned-plugin", + data_designer_specifier=">=0.5.7", + package_version="0.2.0", + ), + _entry( + name="versioned-plugin", + plugin_type="processor", + package_name="data-designer-versioned-plugin", + data_designer_specifier=">=99.0", + package_version="99.0.0", + ), + ], + } + ) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + entry = service.get_entry("versioned-plugin", "local", include_incompatible=True) + + assert entry.package.version == "0.2.0" + + def test_group_entries_by_package_groups_multi_plugin_packages(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") @@ -94,23 +125,57 @@ def test_group_entries_by_package_groups_multi_plugin_packages(tmp_path: Path) - ] -@patch("data_designer.cli.services.plugin_catalog_service.PluginRegistry") -def test_list_installed_plugins_uses_runtime_registry(mock_registry_cls: Mock, tmp_path: Path) -> None: - plugin = Mock() - plugin.name = "installed-plugin" - plugin.plugin_type = PluginType.PROCESSOR - plugin.config_qualified_name = "pkg.config.Config" - plugin.impl_qualified_name = "pkg.impl.Impl" - mock_registry = Mock() - mock_registry.get_plugins.side_effect = lambda plugin_type: [plugin] if plugin_type == PluginType.PROCESSOR else [] - mock_registry_cls.return_value = mock_registry +def test_group_entries_by_package_canonicalizes_package_names(tmp_path: Path) -> None: + repository = _repository_with_catalog(tmp_path) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + entries = [ + PluginCatalogEntry.model_validate( + _entry( + name="hyphen-package", + plugin_type="processor", + package_name="data-designer-shared-package", + data_designer_specifier=">=0.5.7", + ) + ), + PluginCatalogEntry.model_validate( + _entry( + name="underscore-package", + plugin_type="processor", + package_name="data_designer_shared_package", + data_designer_specifier=">=0.5.7", + ) + ), + ] + + grouped_entries = service.group_entries_by_package(entries) + + assert list(grouped_entries) == ["data-designer-shared-package"] + assert [entry.name for entry in grouped_entries["data-designer-shared-package"]] == [ + "hyphen-package", + "underscore-package", + ] + + +@patch("data_designer.cli.services.plugin_catalog_service.importlib.metadata.entry_points") +def test_list_installed_plugins_uses_entry_point_metadata_without_loading_plugins( + mock_entry_points: Mock, + tmp_path: Path, +) -> None: + mock_entry_points.return_value = [ + EntryPoint( + name="installed-plugin", + value="pkg.plugin:plugin", + group="data_designer.plugins", + ) + ] service = PluginCatalogService(PluginTapRepository(tmp_path)) installed = service.list_installed_plugins() assert len(installed) == 1 assert installed[0].name == "installed-plugin" - assert installed[0].plugin_type == PluginType.PROCESSOR + assert installed[0].entry_point_value == "pkg.plugin:plugin" + mock_entry_points.assert_called_once_with(group="data_designer.plugins") def _repository_with_catalog(tmp_path: Path) -> PluginTapRepository: diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index 15abdaf86..abf74b71f 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -119,7 +119,11 @@ def test_install_raises_when_runner_fails() -> None: @patch("data_designer.cli.services.plugin_install_service.PluginRegistry") -def test_verify_entry_point_resets_and_checks_runtime_registry(mock_registry_cls: Mock) -> None: +@patch("data_designer.cli.services.plugin_install_service.importlib.invalidate_caches") +def test_verify_entry_point_invalidates_caches_and_checks_runtime_registry( + mock_invalidate_caches: Mock, + mock_registry_cls: Mock, +) -> None: entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) mock_registry = Mock() mock_registry.plugin_exists.return_value = True @@ -127,6 +131,7 @@ def test_verify_entry_point_resets_and_checks_runtime_registry(mock_registry_cls service = PluginInstallService() assert service.verify_entry_point(entry) is True + mock_invalidate_caches.assert_called_once_with() mock_registry_cls.reset.assert_called_once_with() mock_registry.plugin_exists.assert_called_once_with("text-transform") diff --git a/uv.lock b/uv.lock index 2736484ed..e8048684b 100644 --- a/uv.lock +++ b/uv.lock @@ -3014,11 +3014,11 @@ wheels = [ [[package]] name = "packaging" -version = "25.0" +version = "26.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] [[package]] From 6ffad174da06304598dfbd384b4fc60f76075fd1 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 21:16:09 +0000 Subject: [PATCH 06/34] fix(cli): gate incompatible plugin installs Fetch install targets before compatibility filtering so the controller owns the final --force decision and the incompatible install guard stays reachable. Refs #617 --- .../cli/controllers/plugin_catalog_controller.py | 2 +- .../tests/cli/controllers/test_plugin_catalog_controller.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 8a8357658..ddc9e2c82 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -138,7 +138,7 @@ def run_install( ) -> None: """Install one plugin from a catalog entry.""" tap = self._get_tap_or_exit(tap_alias) - entry = self._get_entry_or_exit(plugin_name, tap.alias, refresh=refresh, include_incompatible=force) + entry = self._get_entry_or_exit(plugin_name, tap.alias, refresh=refresh, include_incompatible=True) compatibility = self.catalog_service.evaluate_compatibility(entry) if not compatibility.is_compatible and not force: diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 2851be8d4..ed7be8f91 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -47,7 +47,7 @@ def test_run_install_dry_run_renders_plan_without_installing( "text-transform", "local", refresh=False, - include_incompatible=False, + include_incompatible=True, ) controller.install_service.install.assert_not_called() controller.install_service.verify_entry_point.assert_not_called() @@ -79,7 +79,7 @@ def test_run_install_blocks_incompatible_plugin_without_force( "text-transform", "local", refresh=False, - include_incompatible=False, + include_incompatible=True, ) controller.install_service.build_install_plan.assert_not_called() mock_print_error.assert_called_once_with("Plugin 'text-transform' is not compatible with this environment") From 4bfb03e1c517719dd9d1e446231bd224d0b75f8e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 21:25:22 +0000 Subject: [PATCH 07/34] style(cli): format plugin catalog files Apply ruff formatting to the plugin command and tap repository tests so CI format checks pass on the PR merge commit. Refs #617 --- .../data-designer/src/data_designer/cli/commands/plugins.py | 4 +++- .../tests/cli/repositories/test_plugin_tap_repository.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugins.py index e58e862f2..0cf576752 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugins.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugins.py @@ -40,7 +40,9 @@ def list_command( def search_command( ctx: typer.Context, - query: str = typer.Argument(help="Keyword, plugin type, package name, source, docs URL, or entry point to search for."), + query: str = typer.Argument( + help="Keyword, plugin type, package name, source, docs URL, or entry point to search for." + ), tap: str | None = typer.Option( None, "--tap", diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py index eeed579b0..a0c609128 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py @@ -254,7 +254,8 @@ def _plugin_entry( "marker": None, }, }, - "source": source or { + "source": source + or { "type": "pypi", "package": package_name, }, From 1d20aae5d648c6b82c7e226826920cdc0225b23f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Thu, 7 May 2026 21:29:05 +0000 Subject: [PATCH 08/34] fix(cli): reject duplicate plugin entry names Key catalog duplicate detection by entry_point.name so distinct catalog entries cannot register the same runtime plugin name. Refs #617 --- .../src/data_designer/cli/plugin_catalog.py | 12 ++++++------ .../cli/repositories/test_plugin_tap_repository.py | 8 +++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index 915ec317b..5c81bd08e 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -215,15 +215,15 @@ def _validate_plugin_catalog_payload(payload: object) -> None: runtime_names: dict[str, tuple[str, str]] = {} for index, raw_plugin in enumerate(plugins): package_name, plugin_name, entry_point_name = _validate_catalog_plugin(raw_plugin, index) - previous = runtime_names.get(plugin_name) + previous = runtime_names.get(entry_point_name) if previous is not None: - previous_package, previous_entry_point = previous + previous_package, previous_plugin_name = previous raise PluginCatalogError( - f"duplicate runtime plugin name {plugin_name!r} from " - f"{previous_package!r} entry point {previous_entry_point!r} and " - f"{package_name!r} entry point {entry_point_name!r}" + f"duplicate runtime plugin name {entry_point_name!r} from " + f"catalog plugin {previous_plugin_name!r} in package {previous_package!r} and " + f"catalog plugin {plugin_name!r} in package {package_name!r}" ) - runtime_names[plugin_name] = (package_name, entry_point_name) + runtime_names[entry_point_name] = (package_name, plugin_name) def _validate_catalog_plugin(raw_plugin: object, index: int) -> tuple[str, str, str]: diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py index a0c609128..85fb332d3 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py @@ -187,8 +187,8 @@ def test_load_catalog_rejects_duplicate_runtime_plugin_names(tmp_path: Path) -> catalog_path = _write_catalog( tmp_path, plugins=[ - _plugin_entry("duplicate", package_name="data-designer-one"), - _plugin_entry("duplicate", package_name="data-designer-two"), + _plugin_entry("catalog-one", package_name="data-designer-one", entry_point_name="duplicate"), + _plugin_entry("catalog-two", package_name="data-designer-two", entry_point_name="duplicate"), ], ) repository = PluginTapRepository(tmp_path) @@ -230,8 +230,10 @@ def _plugin_entry( plugin_name: str, *, package_name: str = "data-designer-text-transform", + entry_point_name: str | None = None, source: dict | None = None, ) -> dict: + runtime_plugin_name = plugin_name if entry_point_name is None else entry_point_name return { "name": plugin_name, "plugin_type": "processor", @@ -243,7 +245,7 @@ def _plugin_entry( }, "entry_point": { "group": "data_designer.plugins", - "name": plugin_name, + "name": runtime_plugin_name, "value": f"{package_name.replace('-', '_')}.plugin:plugin", }, "compatibility": { From f8f866bd772ae8aab0cd8728b098b4cbf1bbb90e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 02:36:26 +0000 Subject: [PATCH 09/34] fix(cli): preserve GitHub tree tap paths --- .../cli/repositories/plugin_tap_repository.py | 4 +++- .../tests/cli/repositories/test_plugin_tap_repository.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py index 2e896a1bb..13fac8fce 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py @@ -257,7 +257,9 @@ def _normalize_tap_url(url: str) -> str: return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}" if len(segments) >= 4 and segments[2] == "tree": ref = segments[3] - return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/catalog/plugins.json" + tap_root = "/".join(segments[4:]) + catalog_path = f"{tap_root}/catalog/plugins.json" if tap_root else "catalog/plugins.json" + return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{catalog_path}" return url diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py index 85fb332d3..dc26c9d69 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py @@ -43,6 +43,14 @@ def test_add_tap_normalizes_github_repository_url(tmp_path: Path) -> None: assert repository.get_tap("research") == tap +def test_add_tap_normalizes_github_tree_url_with_subdirectory(tmp_path: Path) -> None: + repository = PluginTapRepository(tmp_path) + + tap = repository.add_tap("research", "https://github.com/acme/dd-plugins/tree/main/custom-catalog") + + assert tap.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/custom-catalog/catalog/plugins.json" + + def test_tap_aliases_are_case_insensitive(tmp_path: Path) -> None: repository = PluginTapRepository(tmp_path) From 4717aab9192472c86abc486322f2d72aed7d2c8e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 03:27:15 +0000 Subject: [PATCH 10/34] fix(cli): verify plugin entry point names --- .../cli/services/plugin_install_service.py | 19 +++++++---- .../services/test_plugin_install_service.py | 34 ++++++++++++------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 22b39e02f..0c8aa511c 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib +import importlib.metadata import shutil import subprocess import sys @@ -11,8 +12,13 @@ from pathlib import Path from urllib.parse import urlparse -from data_designer.cli.plugin_catalog import InstallPlan, PluginCatalogEntry, PluginSourceInfo, PluginTapConfig -from data_designer.plugins.registry import PluginRegistry +from data_designer.cli.plugin_catalog import ( + PLUGIN_ENTRY_POINT_GROUP, + InstallPlan, + PluginCatalogEntry, + PluginSourceInfo, + PluginTapConfig, +) InstallRunner = Callable[[list[str]], int] @@ -55,11 +61,12 @@ def install(self, plan: InstallPlan) -> None: raise RuntimeError(f"Plugin installer exited with status {return_code}") def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: - """Verify the plugin is discoverable by the runtime PluginRegistry.""" + """Verify the plugin's declared entry point is installed.""" importlib.invalidate_caches() - PluginRegistry.reset() - registry = PluginRegistry() - return registry.plugin_exists(entry.name) + return any( + entry_point.name == entry.entry_point.name + for entry_point in importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP) + ) def _run_subprocess(command: list[str]) -> int: diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index abf74b71f..fdd120dee 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -5,6 +5,7 @@ import sys from pathlib import Path +from types import SimpleNamespace from unittest.mock import Mock, patch import pytest @@ -118,27 +119,36 @@ def test_install_raises_when_runner_fails() -> None: service.install(plan) -@patch("data_designer.cli.services.plugin_install_service.PluginRegistry") +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") @patch("data_designer.cli.services.plugin_install_service.importlib.invalidate_caches") -def test_verify_entry_point_invalidates_caches_and_checks_runtime_registry( +def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( mock_invalidate_caches: Mock, - mock_registry_cls: Mock, + mock_entry_points: Mock, ) -> None: - entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) - mock_registry = Mock() - mock_registry.plugin_exists.return_value = True - mock_registry_cls.return_value = mock_registry + entry = _entry( + source={"type": "pypi", "package": "data-designer-text-transform"}, + plugin_name="text-transform-v2", + entry_point_name="text-transform", + ) + mock_entry_points.return_value = [ + SimpleNamespace(name="other-plugin"), + SimpleNamespace(name="text-transform"), + ] service = PluginInstallService() assert service.verify_entry_point(entry) is True mock_invalidate_caches.assert_called_once_with() - mock_registry_cls.reset.assert_called_once_with() - mock_registry.plugin_exists.assert_called_once_with("text-transform") + mock_entry_points.assert_called_once_with(group="data_designer.plugins") -def _entry(source: dict | None) -> PluginCatalogEntry: +def _entry( + source: dict | None, + *, + plugin_name: str = "text-transform", + entry_point_name: str = "text-transform", +) -> PluginCatalogEntry: payload = { - "name": "text-transform", + "name": plugin_name, "plugin_type": "processor", "description": "Transform text records", "package": { @@ -148,7 +158,7 @@ def _entry(source: dict | None) -> PluginCatalogEntry: }, "entry_point": { "group": "data_designer.plugins", - "name": "text-transform", + "name": entry_point_name, "value": "data_designer_text_transform.plugin:plugin", }, "compatibility": { From 7799e41aca99e487f4b7680e466b35f11d7dcf6b Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 18:27:04 +0000 Subject: [PATCH 11/34] align plugin CLI with catalog schema - adopt catalog terminology for plugin source aliases - parse package-first plugin catalog metadata from the plugin repo - install package requirements with optional catalog indexes --- .../src/data_designer/cli/README.md | 40 +- .../src/data_designer/cli/commands/plugins.py | 110 +++--- .../controllers/plugin_catalog_controller.py | 171 ++++---- .../src/data_designer/cli/main.py | 38 +- .../src/data_designer/cli/plugin_catalog.py | 372 +++++++++--------- ...sitory.py => plugin_catalog_repository.py} | 178 +++++---- .../cli/services/plugin_catalog_service.py | 110 +++--- .../cli/services/plugin_install_service.py | 101 ++--- .../cli/commands/test_plugins_command.py | 30 +- .../test_plugin_catalog_controller.py | 105 ++--- .../test_plugin_catalog_repository.py | 306 ++++++++++++++ .../test_plugin_tap_repository.py | 275 ------------- .../services/test_plugin_catalog_service.py | 118 +++--- .../services/test_plugin_install_service.py | 156 ++++---- 14 files changed, 1086 insertions(+), 1024 deletions(-) rename packages/data-designer/src/data_designer/cli/repositories/{plugin_tap_repository.py => plugin_catalog_repository.py} (58%) create mode 100644 packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py delete mode 100644 packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 338912ea4..66d69c442 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -1,13 +1,13 @@ # 🎨 NeMo Data Designer CLI -This directory contains the Command-Line Interface (CLI) for configuring model providers, model configurations, managed assets, and plugin tap catalogs used in Data Designer. +This directory contains the Command-Line Interface (CLI) for configuring model providers, model configurations, managed assets, and plugin catalogs used in Data Designer. ## Overview The CLI provides an interactive interface for managing: - **Model Providers**: LLM API endpoints (NVIDIA, OpenAI, Anthropic, custom providers) - **Model Configs**: Specific model configurations with inference parameters -- **Plugin Taps**: Catalog aliases for discovering Data Designer plugin packages +- **Plugin Catalogs**: Catalog aliases for discovering Data Designer plugin packages - **Plugin Installs**: Safe install-plan rendering, package-manager execution, and entry point verification Configuration files are stored in `~/.data-designer/` by default and can be referenced by Data Designer workflows. @@ -54,7 +54,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `list.py`: List current configurations - `models.py`: Configure models - `providers.py`: Configure providers - - `plugins.py`: Discover and install plugins from tap catalogs + - `plugins.py`: Discover and install plugins from catalogs - `reset.py`: Reset/delete configurations #### 2. **Controllers** (`controllers/`) @@ -67,7 +67,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - **Files**: - `model_controller.py`: Orchestrates model configuration workflows - `provider_controller.py`: Orchestrates provider configuration workflows - - `plugin_catalog_controller.py`: Orchestrates plugin catalog, tap, and install workflows + - `plugin_catalog_controller.py`: Orchestrates plugin catalog, catalog, and install workflows **Key Features**: - **Associated Resource Management**: When deleting a provider, the controller checks for associated models and prompts the user to delete them together @@ -107,7 +107,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `base.py`: Abstract base repository with common operations - `model_repository.py`: Model configuration persistence - `provider_repository.py`: Provider persistence - - `plugin_tap_repository.py`: Plugin tap aliases, catalog fetching, and URL-keyed catalog cache + - `plugin_catalog_repository.py`: Plugin catalog aliases, catalog fetching, and URL-keyed catalog cache **Base Repository Pattern**: ```python @@ -213,21 +213,25 @@ model_configs: max_parallel_requests: 4 ``` -### `~/.data-designer/plugin_taps.yaml` +### `~/.data-designer/plugin_catalogs.yaml` -Stores user-added plugin tap aliases. The built-in NVIDIA tap is always available and is not written to this file. Set `DATA_DESIGNER_DEFAULT_PLUGIN_TAP_URL` to repoint the built-in tap for QA or staging. +Stores user-added plugin catalog aliases. The built-in NVIDIA catalog points at +`https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`, is +always available, and is not written to this file. Set +`DATA_DESIGNER_DEFAULT_PLUGIN_CATALOG_URL` to repoint the built-in catalog for QA or +staging. ```yaml -taps: +catalogs: - alias: research url: https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json trusted: false cache_ttl_seconds: 86400 ``` -### `~/.data-designer/plugin-tap-cache/` +### `~/.data-designer/plugin-catalog-cache/` -Stores fetched plugin catalog payloads as JSON cache files keyed by tap alias and URL hash. This prevents a re-pointed alias from serving stale catalog data from a previous URL. +Stores fetched plugin catalog payloads as JSON cache files keyed by catalog alias and URL hash. This prevents a re-pointed alias from serving stale catalog data from a previous URL. ## Usage Examples @@ -275,25 +279,25 @@ data-designer config reset ### Discover and Install Plugins ```bash -# List compatible plugins from the default NVIDIA tap +# List compatible plugins from the default NVIDIA catalog data-designer plugins list -# Search a specific tap catalog -data-designer plugins --tap research search transform +# Search a specific catalog +data-designer plugins --catalog research search transform # Show metadata, compatibility, docs, and exact install command data-designer plugins info text-transform -# Install from a trusted or user-added tap and verify entry point discovery +# Install a plugin package from a catalog and verify declared entry point discovery data-designer plugins install text-transform --yes # Preview the install command without mutating the environment data-designer plugins install text-transform --dry-run -# Add and manage tap aliases -data-designer plugins taps add research https://github.com/acme/dd-plugins -data-designer plugins taps list -data-designer plugins taps remove research +# Add and manage catalog aliases +data-designer plugins catalogs add research https://github.com/acme/dd-plugins +data-designer plugins catalogs list +data-designer plugins catalogs remove research # List installed plugin entry points without importing plugin modules data-designer plugins installed diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugins.py index 0cf576752..17ab6a4ab 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugins.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugins.py @@ -13,15 +13,15 @@ def list_command( ctx: typer.Context, - tap: str | None = typer.Option( + catalog: str | None = typer.Option( None, - "--tap", - help="Plugin tap alias to read. Can also be provided before the subcommand.", + "--catalog", + help="Plugin catalog alias to read. Can also be provided before the subcommand.", ), refresh: bool = typer.Option( False, "--refresh", - help="Fetch the tap catalog even when a fresh cache entry exists.", + help="Fetch the catalog even when a fresh cache entry exists.", ), include_incompatible: bool = typer.Option( False, @@ -29,10 +29,10 @@ def list_command( help="Show plugins that do not satisfy the local Python or Data Designer version.", ), ) -> None: - """List discoverable Data Designer plugins from a tap catalog.""" + """List discoverable Data Designer plugins from a catalog.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_list( - tap_alias=_resolve_tap_alias(ctx, tap), + catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, include_incompatible=include_incompatible, ) @@ -41,17 +41,17 @@ def list_command( def search_command( ctx: typer.Context, query: str = typer.Argument( - help="Keyword, plugin type, package name, source, docs URL, or entry point to search for." + help="Keyword, plugin type, package name, requirement, docs URL, or entry point to search for." ), - tap: str | None = typer.Option( + catalog: str | None = typer.Option( None, - "--tap", - help="Plugin tap alias to search. Can also be provided before the subcommand.", + "--catalog", + help="Plugin catalog alias to search. Can also be provided before the subcommand.", ), refresh: bool = typer.Option( False, "--refresh", - help="Fetch the tap catalog even when a fresh cache entry exists.", + help="Fetch the catalog even when a fresh cache entry exists.", ), include_incompatible: bool = typer.Option( False, @@ -59,11 +59,11 @@ def search_command( help="Search plugins that do not satisfy the local Python or Data Designer version.", ), ) -> None: - """Search discoverable Data Designer plugins from a tap catalog.""" + """Search discoverable Data Designer plugins from a catalog.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_search( query, - tap_alias=_resolve_tap_alias(ctx, tap), + catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, include_incompatible=include_incompatible, ) @@ -71,39 +71,39 @@ def search_command( def info_command( ctx: typer.Context, - plugin_name: str = typer.Argument(help="Plugin name from the tap catalog."), - tap: str | None = typer.Option( + plugin_name: str = typer.Argument(help="Runtime plugin name or package name from the catalog."), + catalog: str | None = typer.Option( None, - "--tap", - help="Plugin tap alias to read. Can also be provided before the subcommand.", + "--catalog", + help="Plugin catalog alias to read. Can also be provided before the subcommand.", ), refresh: bool = typer.Option( False, "--refresh", - help="Fetch the tap catalog even when a fresh cache entry exists.", + help="Fetch the catalog even when a fresh cache entry exists.", ), ) -> None: - """Show metadata, compatibility, docs, and install plan for one plugin.""" + """Show metadata, compatibility, docs, and install plan for one plugin package.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_info( plugin_name, - tap_alias=_resolve_tap_alias(ctx, tap), + catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, ) def install_command( ctx: typer.Context, - plugin_name: str = typer.Argument(help="Plugin name from the tap catalog."), - tap: str | None = typer.Option( + plugin_name: str = typer.Argument(help="Runtime plugin name or package name from the catalog."), + catalog: str | None = typer.Option( None, - "--tap", - help="Plugin tap alias to install from. Can also be provided before the subcommand.", + "--catalog", + help="Plugin catalog alias to install from. Can also be provided before the subcommand.", ), refresh: bool = typer.Option( False, "--refresh", - help="Fetch the tap catalog even when a fresh cache entry exists.", + help="Fetch the catalog even when a fresh cache entry exists.", ), manager: str = typer.Option( "auto", @@ -132,7 +132,7 @@ def install_command( controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_install( plugin_name, - tap_alias=_resolve_tap_alias(ctx, tap), + catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, manager=manager, yes=yes, @@ -143,26 +143,28 @@ def install_command( def installed_command(ctx: typer.Context) -> None: """List installed Data Designer plugin entry points.""" - _warn_if_parent_tap_unused(ctx, "installed plugins are discovered from the current Python environment") + _warn_if_parent_catalog_unused(ctx, "installed plugins are discovered from the current Python environment") controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_installed() -def taps_list_command(ctx: typer.Context) -> None: - """List configured plugin taps.""" - _warn_if_parent_tap_unused(ctx, "tap management commands operate on aliases directly") +def catalogs_list_command(ctx: typer.Context) -> None: + """List configured plugin catalogs.""" + _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) - controller.run_taps_list() + controller.run_catalogs_list() -def taps_add_command( +def catalogs_add_command( ctx: typer.Context, - alias: str = typer.Argument(help="Local alias for the plugin tap."), - url: str = typer.Argument(help="Tap repository URL, catalog URL, local catalog file, or local tap directory."), + alias: str = typer.Argument(help="Local alias for the plugin catalog."), + url: str = typer.Argument( + help="Catalog repository URL, catalog URL, local catalog file, or local catalog directory." + ), trusted: bool = typer.Option( False, "--trusted", - help="Mark the tap as trusted for install-plan display and confirmations.", + help="Mark the catalog as trusted for install-plan display and confirmations.", ), cache_ttl_seconds: int = typer.Option( 24 * 60 * 60, @@ -171,10 +173,10 @@ def taps_add_command( help="Seconds before cached catalog metadata is refreshed. Use 0 to always refresh.", ), ) -> None: - """Add a plugin tap alias.""" - _warn_if_parent_tap_unused(ctx, "tap management commands operate on aliases directly") + """Add a plugin catalog alias.""" + _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) - controller.run_taps_add( + controller.run_catalogs_add( alias=alias, url=url, trusted=trusted, @@ -182,36 +184,36 @@ def taps_add_command( ) -def taps_remove_command( +def catalogs_remove_command( ctx: typer.Context, - alias: str = typer.Argument(help="Plugin tap alias to remove."), + alias: str = typer.Argument(help="Plugin catalog alias to remove."), ) -> None: - """Remove a plugin tap alias.""" - _warn_if_parent_tap_unused(ctx, "tap management commands operate on aliases directly") + """Remove a plugin catalog alias.""" + _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) - controller.run_taps_remove(alias=alias) + controller.run_catalogs_remove(alias=alias) -def _resolve_tap_alias(ctx: typer.Context, tap_alias: str | None) -> str | None: - if tap_alias is not None: - return tap_alias +def _resolve_catalog_alias(ctx: typer.Context, catalog_alias: str | None) -> str | None: + if catalog_alias is not None: + return catalog_alias - return _parent_tap_alias(ctx) + return _parent_catalog_alias(ctx) -def _parent_tap_alias(ctx: typer.Context) -> str | None: - """Return --tap from the plugins parent command when present.""" +def _parent_catalog_alias(ctx: typer.Context) -> str | None: + """Return --catalog from the plugins parent command when present.""" parent = ctx.parent while parent is not None: - candidate = parent.params.get("tap") if parent.params else None + candidate = parent.params.get("catalog") if parent.params else None if isinstance(candidate, str) and candidate: return candidate parent = parent.parent return None -def _warn_if_parent_tap_unused(ctx: typer.Context, reason: str) -> None: - tap_alias = _parent_tap_alias(ctx) - if tap_alias is not None: - print_info(f"Ignoring --tap {tap_alias!r}; {reason}.") +def _warn_if_parent_catalog_unused(ctx: typer.Context, reason: str) -> None: + catalog_alias = _parent_catalog_alias(ctx) + if catalog_alias is not None: + print_info(f"Ignoring --catalog {catalog_alias!r}; {reason}.") diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index ddc9e2c82..eb26ebb49 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -11,14 +11,14 @@ from rich.table import Table from data_designer.cli.plugin_catalog import ( - DEFAULT_PLUGIN_TAP_ALIAS, - PLUGIN_TAP_ALIAS_PATTERN, + DEFAULT_PLUGIN_CATALOG_ALIAS, + PLUGIN_CATALOG_ALIAS_PATTERN, CompatibilityResult, InstalledPluginInfo, + PluginCatalogConfig, PluginCatalogEntry, - PluginTapConfig, ) -from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository +from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository from data_designer.cli.services.plugin_catalog_service import PluginCatalogService from data_designer.cli.services.plugin_install_service import PluginInstallService from data_designer.cli.ui import ( @@ -35,31 +35,31 @@ class PluginCatalogController: - """Controller for plugin catalog, tap, and install workflows. + """Controller for plugin catalog, catalog, and install workflows. Catalog browsing and environment mutation intentionally use separate services so - read-only tap operations stay decoupled from package-manager execution. + read-only catalog operations stay decoupled from package-manager execution. """ def __init__(self, config_dir: Path) -> None: self.config_dir = config_dir - self.tap_repository = PluginTapRepository(config_dir) - self.catalog_service = PluginCatalogService(self.tap_repository) + self.catalog_repository = PluginCatalogRepository(config_dir) + self.catalog_service = PluginCatalogService(self.catalog_repository) self.install_service = PluginInstallService() def run_list( self, *, - tap_alias: str | None = None, + catalog_alias: str | None = None, refresh: bool = False, include_incompatible: bool = False, ) -> None: - """List plugins from a tap catalog.""" - tap = self._get_tap_or_exit(tap_alias) - entries = self._list_entries_or_exit(tap.alias, refresh=refresh, include_incompatible=include_incompatible) + """List plugins from a catalog.""" + catalog = self._get_catalog_or_exit(catalog_alias) + entries = self._list_entries_or_exit(catalog.alias, refresh=refresh, include_incompatible=include_incompatible) print_header("Data Designer Plugins") - print_info(f"Tap: {tap.alias} ({tap.url})") + print_info(f"Catalog: {catalog.alias} ({catalog.url})") console.print() if not entries: @@ -72,16 +72,16 @@ def run_search( self, query: str, *, - tap_alias: str | None = None, + catalog_alias: str | None = None, refresh: bool = False, include_incompatible: bool = False, ) -> None: - """Search plugins from a tap catalog.""" - tap = self._get_tap_or_exit(tap_alias) + """Search plugins from a catalog.""" + catalog = self._get_catalog_or_exit(catalog_alias) try: entries = self.catalog_service.search_entries( query, - tap.alias, + catalog.alias, refresh=refresh, include_incompatible=include_incompatible, ) @@ -90,7 +90,7 @@ def run_search( raise typer.Exit(code=1) print_header("Data Designer Plugin Search") - print_info(f"Tap: {tap.alias} ({tap.url})") + print_info(f"Catalog: {catalog.alias} ({catalog.url})") print_info(f"Query: {query}") console.print() @@ -104,32 +104,55 @@ def run_info( self, plugin_name: str, *, - tap_alias: str | None = None, + catalog_alias: str | None = None, refresh: bool = False, ) -> None: """Show full metadata for one plugin.""" - tap = self._get_tap_or_exit(tap_alias) - entry = self._get_entry_or_exit(plugin_name, tap.alias, refresh=refresh) + catalog = self._get_catalog_or_exit(catalog_alias) + entry = self._get_entry_or_exit(plugin_name, catalog.alias, refresh=refresh) + package_entries = self.catalog_service.get_package_entries( + entry.package.name, + catalog.alias, + refresh=refresh, + include_incompatible=True, + ) or [entry] compatibility = self.catalog_service.evaluate_compatibility(entry) - print_header(f"Plugin: {entry.name}") - print_info(f"Tap: {tap.alias} ({tap.url})") + print_header(f"Plugin Package: {entry.package.name}") + print_info(f"Catalog: {catalog.alias} ({catalog.url})") + console.print(f" Runtime plugins: [bold]{_format_runtime_plugins(package_entries)}[/bold]") self._display_compatibility(compatibility) try: - plan = self.install_service.build_install_plan(entry, tap) + plan = self.install_service.build_install_plan(entry, catalog) + console.print(f" Requirement: [bold]{entry.install.requirement}[/bold]") + if entry.install.index_url is not None: + console.print(f" Index URL: [bold]{entry.install.index_url}[/bold]") console.print(f" Install command: [bold]{shlex.join(plan.command)}[/bold]") except ValueError as e: print_warning(str(e)) console.print() - display_config_preview(entry.model_dump(mode="json", exclude_none=True), "Plugin Metadata") + display_config_preview( + { + "package": entry.package.name, + "install": entry.install.model_dump(mode="json", exclude_none=True), + "compatibility": ( + entry.compatibility.model_dump(mode="json", exclude_none=True) + if entry.compatibility is not None + else None + ), + "docs": entry.docs.model_dump(mode="json", exclude_none=True) if entry.docs is not None else None, + "plugins": [plugin.model_dump(mode="json", exclude_none=True) for plugin in package_entries], + }, + "Plugin Metadata", + ) def run_install( self, plugin_name: str, *, - tap_alias: str | None = None, + catalog_alias: str | None = None, refresh: bool = False, manager: str = "auto", yes: bool = False, @@ -137,8 +160,14 @@ def run_install( force: bool = False, ) -> None: """Install one plugin from a catalog entry.""" - tap = self._get_tap_or_exit(tap_alias) - entry = self._get_entry_or_exit(plugin_name, tap.alias, refresh=refresh, include_incompatible=True) + catalog = self._get_catalog_or_exit(catalog_alias) + entry = self._get_entry_or_exit(plugin_name, catalog.alias, refresh=refresh, include_incompatible=True) + package_entries = self.catalog_service.get_package_entries( + entry.package.name, + catalog.alias, + refresh=refresh, + include_incompatible=True, + ) or [entry] compatibility = self.catalog_service.evaluate_compatibility(entry) if not compatibility.is_compatible and not force: @@ -148,21 +177,24 @@ def run_install( raise typer.Exit(code=1) try: - plan = self.install_service.build_install_plan(entry, tap, manager=manager) + plan = self.install_service.build_install_plan(entry, catalog, manager=manager) except ValueError as e: print_error(f"Failed to build plugin install plan: {e}") raise typer.Exit(code=1) print_header("Install Data Designer Plugin") - console.print(f" Plugin: [bold]{entry.name}[/bold]") - console.print(f" Tap: [bold]{tap.alias}[/bold] ({tap.url})") - console.print(f" Source: [bold]{plan.source_description}[/bold]") + console.print(f" Package: [bold]{entry.package.name}[/bold]") + console.print(f" Runtime plugins: [bold]{_format_runtime_plugins(package_entries)}[/bold]") + console.print(f" Catalog: [bold]{catalog.alias}[/bold] ({catalog.url})") + console.print(f" Requirement: [bold]{entry.install.requirement}[/bold]") + if entry.install.index_url is not None: + console.print(f" Index URL: [bold]{entry.install.index_url}[/bold]") console.print(f" Command: [bold]{shlex.join(plan.command)}[/bold]") self._display_compatibility(compatibility) - if not tap.trusted: + if not catalog.trusted: print_warning( - "This tap is not marked trusted. Plugin installation executes Python package code from the source above." + "This catalog is not marked trusted. Plugin installation executes Python package code from the requirement above." ) if dry_run: @@ -179,11 +211,12 @@ def run_install( print_error(str(e)) raise typer.Exit(code=1) - if self.install_service.verify_entry_point(entry): - print_success(f"Plugin {entry.name!r} installed and discovered") + if self.install_service.verify_entry_points(package_entries): + print_success(f"Plugin package {entry.package.name!r} installed and discovered") else: print_warning( - f"Plugin {entry.name!r} was installed, but Data Designer did not discover its entry point. " + f"Plugin package {entry.package.name!r} was installed, but Data Designer did not discover every " + "declared entry point. " "Restart the shell or check the package entry point metadata." ) @@ -196,26 +229,26 @@ def run_installed(self) -> None: return self._display_installed_plugins(installed_plugins) - def run_taps_list(self) -> None: - """List configured plugin taps.""" - print_header("Data Designer Plugin Taps") - taps = self.catalog_service.list_taps() - table = Table(title="Plugin Taps", border_style=NordColor.NORD8.value) + def run_catalogs_list(self) -> None: + """List configured plugin catalogs.""" + print_header("Data Designer Plugin Catalogs") + catalogs = self.catalog_service.list_catalogs() + table = Table(title="Plugin Catalogs", border_style=NordColor.NORD8.value) table.add_column("Alias", style=NordColor.NORD14.value, no_wrap=True) table.add_column("URL", style=NordColor.NORD4.value) table.add_column("Trusted", style=NordColor.NORD13.value, justify="center") table.add_column("Cache TTL", style=NordColor.NORD9.value, justify="right") - for tap in taps: + for catalog in catalogs: table.add_row( - tap.alias, - tap.url, - "yes" if tap.trusted else "no", - f"{tap.cache_ttl_seconds}s", + catalog.alias, + catalog.url, + "yes" if catalog.trusted else "no", + f"{catalog.cache_ttl_seconds}s", ) console.print(table) - def run_taps_add( + def run_catalogs_add( self, *, alias: str, @@ -223,9 +256,9 @@ def run_taps_add( trusted: bool, cache_ttl_seconds: int, ) -> None: - """Add a plugin tap alias.""" + """Add a plugin catalog alias.""" try: - tap = self.catalog_service.add_tap( + catalog = self.catalog_service.add_catalog( alias, url, trusted=trusted, @@ -233,43 +266,43 @@ def run_taps_add( ) except ValidationError as e: if any(tuple(error["loc"]) == ("alias",) for error in e.errors()): - print_error(f"Invalid tap alias {alias!r}: must match `{PLUGIN_TAP_ALIAS_PATTERN}`") + print_error(f"Invalid catalog alias {alias!r}: must match `{PLUGIN_CATALOG_ALIAS_PATTERN}`") else: - print_error(f"Invalid plugin tap configuration: {e}") + print_error(f"Invalid plugin catalog configuration: {e}") raise typer.Exit(code=1) except Exception as e: - print_error(f"Failed to add plugin tap: {e}") + print_error(f"Failed to add plugin catalog: {e}") raise typer.Exit(code=1) - print_success(f"Plugin tap {tap.alias!r} added") - print_info(f"Catalog: {tap.url}") + print_success(f"Plugin catalog {catalog.alias!r} added") + print_info(f"Catalog: {catalog.url}") - def run_taps_remove(self, *, alias: str) -> None: - """Remove a plugin tap alias.""" + def run_catalogs_remove(self, *, alias: str) -> None: + """Remove a plugin catalog alias.""" try: - self.catalog_service.remove_tap(alias) + self.catalog_service.remove_catalog(alias) except Exception as e: - print_error(f"Failed to remove plugin tap: {e}") + print_error(f"Failed to remove plugin catalog: {e}") raise typer.Exit(code=1) - print_success(f"Plugin tap {alias!r} removed") + print_success(f"Plugin catalog {alias!r} removed") - def _get_tap_or_exit(self, tap_alias: str | None) -> PluginTapConfig: + def _get_catalog_or_exit(self, catalog_alias: str | None) -> PluginCatalogConfig: try: - return self.catalog_service.get_tap(tap_alias or DEFAULT_PLUGIN_TAP_ALIAS) + return self.catalog_service.get_catalog(catalog_alias or DEFAULT_PLUGIN_CATALOG_ALIAS) except ValueError as e: print_error(str(e)) raise typer.Exit(code=1) def _list_entries_or_exit( self, - tap_alias: str, + catalog_alias: str, *, refresh: bool, include_incompatible: bool, ) -> list[PluginCatalogEntry]: try: return self.catalog_service.list_entries( - tap_alias, + catalog_alias, refresh=refresh, include_incompatible=include_incompatible, ) @@ -280,7 +313,7 @@ def _list_entries_or_exit( def _get_entry_or_exit( self, plugin_name: str, - tap_alias: str, + catalog_alias: str, *, refresh: bool, include_incompatible: bool = True, @@ -288,7 +321,7 @@ def _get_entry_or_exit( try: return self.catalog_service.get_entry( plugin_name, - tap_alias, + catalog_alias, refresh=refresh, include_incompatible=include_incompatible, ) @@ -301,7 +334,6 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) table.add_column("Type", style=NordColor.NORD9.value, no_wrap=True) table.add_column("Package", style=NordColor.NORD4.value, no_wrap=True) - table.add_column("Version", style=NordColor.NORD15.value, no_wrap=True) table.add_column("Compatible", style=NordColor.NORD13.value, no_wrap=True) table.add_column("Docs", style=NordColor.NORD7.value) @@ -312,7 +344,6 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: entry.name, entry.plugin_type.value, entry.package.name, - entry.package.version or "", "yes" if compatibility.is_compatible else "no", docs_url, ) @@ -340,3 +371,7 @@ def _display_compatibility(compatibility: CompatibilityResult) -> None: console.print(" Compatibility: [bold yellow]not compatible[/bold yellow]") for reason in compatibility.reasons: console.print(f" - {reason}") + + +def _format_runtime_plugins(entries: list[PluginCatalogEntry]) -> str: + return ", ".join(f"{entry.name} ({entry.plugin_type.value})" for entry in entries) diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index 993d34a07..c1548aa3a 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -144,28 +144,28 @@ def _is_version_request(args: list[str]) -> bool: # Create plugins command group plugins_app = typer.Typer( name="plugins", - help="Discover and install Data Designer plugins from tap catalogs", + help="Discover and install Data Designer plugins from catalogs", cls=create_lazy_typer_group( { "list": { "module": f"{_CMD}.plugins", "attr": "list_command", - "help": "List plugins from a tap catalog", + "help": "List plugins from a catalog", }, "search": { "module": f"{_CMD}.plugins", "attr": "search_command", - "help": "Search plugins from a tap catalog", + "help": "Search plugins from a catalog", }, "info": { "module": f"{_CMD}.plugins", "attr": "info_command", - "help": "Show plugin metadata and install plan", + "help": "Show plugin package metadata and install plan", }, "install": { "module": f"{_CMD}.plugins", "attr": "install_command", - "help": "Install a plugin package and verify discovery", + "help": "Install a plugin package and verify runtime discovery", }, "installed": { "module": f"{_CMD}.plugins", @@ -180,34 +180,34 @@ def _is_version_request(args: list[str]) -> bool: @plugins_app.callback() def plugins_callback( - tap: str | None = typer.Option( + catalog: str | None = typer.Option( None, - "--tap", - help="Plugin tap alias to use for catalog commands.", + "--catalog", + help="Plugin catalog alias to use for catalog commands.", ), ) -> None: - _ = tap + _ = catalog -plugin_taps_app = typer.Typer( - name="taps", - help="Manage plugin tap aliases", +plugin_catalogs_app = typer.Typer( + name="catalogs", + help="Manage plugin catalog aliases", cls=create_lazy_typer_group( { "list": { "module": f"{_CMD}.plugins", - "attr": "taps_list_command", - "help": "List configured plugin taps", + "attr": "catalogs_list_command", + "help": "List configured plugin catalogs", }, "add": { "module": f"{_CMD}.plugins", - "attr": "taps_add_command", - "help": "Add a plugin tap alias", + "attr": "catalogs_add_command", + "help": "Add a plugin catalog alias", }, "remove": { "module": f"{_CMD}.plugins", - "attr": "taps_remove_command", - "help": "Remove a plugin tap alias", + "attr": "catalogs_remove_command", + "help": "Remove a plugin catalog alias", }, } ), @@ -240,7 +240,7 @@ def _build_agent_lazy_group(prefix: str) -> dict[str, dict[str, str]]: ) agent_app.add_typer(agent_state_app, name="state") -plugins_app.add_typer(plugin_taps_app, name="taps") +plugins_app.add_typer(plugin_catalogs_app, name="catalogs") # Add setup command groups app.add_typer(config_app, name="config", rich_help_panel="Setup") diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index 5c81bd08e..b9a864e71 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -4,50 +4,46 @@ from __future__ import annotations import os -import re from dataclasses import dataclass from urllib.parse import urlparse from packaging.requirements import InvalidRequirement, Requirement from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import InvalidName, canonicalize_name -from packaging.version import InvalidVersion, Version from pydantic import BaseModel, ConfigDict, Field from data_designer.plugins.plugin import PluginType -DEFAULT_PLUGIN_TAP_ALIAS = "nvidia" -DEFAULT_PLUGIN_TAP_URL = "https://raw.githubusercontent.com/NVIDIA-NeMo/DataDesignerPlugins/main/catalog/plugins.json" -DEFAULT_PLUGIN_TAP_URL_ENV_VAR = "DATA_DESIGNER_DEFAULT_PLUGIN_TAP_URL" -PLUGIN_TAPS_FILE_NAME = "plugin_taps.yaml" -PLUGIN_TAP_CACHE_DIR_NAME = "plugin-tap-cache" -PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS = 24 * 60 * 60 +DEFAULT_PLUGIN_CATALOG_ALIAS = "nvidia" +DEFAULT_PLUGIN_CATALOG_URL = "https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json" +DEFAULT_PLUGIN_CATALOG_URL_ENV_VAR = "DATA_DESIGNER_DEFAULT_PLUGIN_CATALOG_URL" +PLUGIN_CATALOGS_FILE_NAME = "plugin_catalogs.yaml" +PLUGIN_CATALOG_CACHE_DIR_NAME = "plugin-catalog-cache" +PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS = 24 * 60 * 60 MAX_PLUGIN_CATALOG_SIZE_BYTES = 1 * 1024 * 1024 PLUGIN_CATALOG_SCHEMA_VERSION = 2 -SUPPORTED_PLUGIN_CATALOG_SCHEMA_VERSIONS = {PLUGIN_CATALOG_SCHEMA_VERSION} -PLUGIN_TAP_ALIAS_PATTERN = r"^[A-Za-z0-9_.-]+$" +PLUGIN_CATALOG_ALIAS_PATTERN = r"^[A-Za-z0-9_.-]+$" DATA_DESIGNER_DISTRIBUTION_NAME = "data-designer" PLUGIN_ENTRY_POINT_GROUP = "data_designer.plugins" -CATALOG_DOCUMENT_KEYS = {"plugins", "schema_version"} -CATALOG_PLUGIN_KEYS = { +PYPI_SIMPLE_INDEX_URL = "https://pypi.org/simple/" +CATALOG_DOCUMENT_KEYS = {"packages", "schema_version"} +CATALOG_PACKAGE_KEYS = { "compatibility", "description", "docs", - "entry_point", + "install", "name", - "package", - "plugin_type", - "source", + "plugins", } -CATALOG_PACKAGE_KEYS = {"name", "path", "version"} +CATALOG_PLUGIN_KEYS = {"entry_point", "name", "plugin_type"} CATALOG_ENTRY_POINT_KEYS = {"group", "name", "value"} CATALOG_COMPATIBILITY_KEYS = {"data_designer", "python"} CATALOG_PYTHON_COMPATIBILITY_KEYS = {"specifier"} CATALOG_DATA_DESIGNER_COMPATIBILITY_KEYS = {"marker", "requirement", "specifier"} CATALOG_DOCS_KEYS = {"url"} +CATALOG_INSTALL_REQUIRED_KEYS = {"requirement"} +CATALOG_INSTALL_OPTIONAL_KEYS = {"index_url"} SUPPORTED_PLUGIN_TYPE_VALUES = {plugin_type.value for plugin_type in PluginType} -PACKAGE_PATH_ROOT = "plugins" -PACKAGE_PATH_SEGMENT_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") class PluginCatalogError(ValueError): @@ -65,7 +61,7 @@ class PluginCompatibilityTarget(BaseModel): class PluginCompatibility(BaseModel): - """Compatibility requirements declared by a catalog entry.""" + """Compatibility requirements declared by a catalog package.""" model_config = ConfigDict(extra="allow") @@ -74,13 +70,11 @@ class PluginCompatibility(BaseModel): class PluginPackageInfo(BaseModel): - """Python package metadata for a catalog entry.""" + """Python distribution metadata for a catalog entry.""" model_config = ConfigDict(extra="allow") name: str - version: str | None = None - path: str | None = None class PluginEntryPointInfo(BaseModel): @@ -88,27 +82,22 @@ class PluginEntryPointInfo(BaseModel): model_config = ConfigDict(extra="allow") - group: str = "data_designer.plugins" + group: str = PLUGIN_ENTRY_POINT_GROUP name: str value: str -class PluginSourceInfo(BaseModel): - """Install source metadata for a catalog entry.""" +class PluginInstallInfo(BaseModel): + """Resolver-native install metadata for a catalog package.""" model_config = ConfigDict(extra="allow") - type: str - package: str | None = None - url: str | None = None - ref: str | None = None - path: str | None = None - subdirectory: str | None = None - editable: bool | None = None + requirement: str + index_url: str | None = None class PluginDocsInfo(BaseModel): - """Documentation metadata for a catalog entry.""" + """Documentation metadata for a catalog package.""" model_config = ConfigDict(extra="allow") @@ -116,7 +105,7 @@ class PluginDocsInfo(BaseModel): class PluginCatalogEntry(BaseModel): - """One discoverable Data Designer plugin entry from a tap catalog.""" + """One discoverable runtime plugin entry from a catalog package.""" model_config = ConfigDict(extra="allow") @@ -124,34 +113,84 @@ class PluginCatalogEntry(BaseModel): plugin_type: PluginType description: str = "" package: PluginPackageInfo + install: PluginInstallInfo entry_point: PluginEntryPointInfo compatibility: PluginCompatibility | None = None - source: PluginSourceInfo | None = None docs: PluginDocsInfo | None = None +class PluginCatalogRuntimePlugin(BaseModel): + """Runtime plugin metadata nested under one catalog package.""" + + model_config = ConfigDict(extra="allow") + + name: str + plugin_type: PluginType + entry_point: PluginEntryPointInfo + + +class PluginCatalogPackage(BaseModel): + """One installable package from a package-first plugin catalog.""" + + model_config = ConfigDict(extra="allow") + + name: str + description: str = "" + install: PluginInstallInfo + compatibility: PluginCompatibility | None = None + docs: PluginDocsInfo | None = None + plugins: list[PluginCatalogRuntimePlugin] = Field(default_factory=list) + + def entries(self) -> list[PluginCatalogEntry]: + """Flatten nested runtime plugins while preserving package-level metadata.""" + package = PluginPackageInfo(name=self.name) + return [ + PluginCatalogEntry( + name=plugin.name, + plugin_type=plugin.plugin_type, + description=self.description, + package=package, + install=self.install, + entry_point=plugin.entry_point, + compatibility=self.compatibility, + docs=self.docs, + ) + for plugin in self.plugins + ] + + class PluginCatalog(BaseModel): - """Versioned plugin tap catalog.""" + """Versioned plugin catalog.""" model_config = ConfigDict(extra="allow") schema_version: int - plugins: list[PluginCatalogEntry] = Field(default_factory=list) + packages: list[PluginCatalogPackage] = Field(default_factory=list) + @property + def entries(self) -> list[PluginCatalogEntry]: + """Return the runtime plugin entries described by every package.""" + return [entry for package in self.packages for entry in package.entries()] -class PluginTapConfig(BaseModel): - """Persisted tap configuration.""" + @property + def plugins(self) -> list[PluginCatalogEntry]: + """Backward-compatible alias for flattened runtime plugin entries.""" + return self.entries - alias: str = Field(pattern=PLUGIN_TAP_ALIAS_PATTERN) + +class PluginCatalogConfig(BaseModel): + """Persisted catalog configuration.""" + + alias: str = Field(pattern=PLUGIN_CATALOG_ALIAS_PATTERN) url: str trusted: bool = False - cache_ttl_seconds: int = Field(default=PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, ge=0) + cache_ttl_seconds: int = Field(default=PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, ge=0) -class PluginTapRegistry(BaseModel): - """Persisted collection of user-configured plugin taps.""" +class PluginCatalogRegistry(BaseModel): + """Persisted collection of user-configured plugin catalogs.""" - taps: list[PluginTapConfig] = Field(default_factory=list) + catalogs: list[PluginCatalogConfig] = Field(default_factory=list) @dataclass(frozen=True) @@ -164,15 +203,15 @@ class CompatibilityResult: @dataclass(frozen=True) class InstallPlan: - """Resolved package-manager command for installing one plugin entry.""" + """Resolved package-manager command for installing one plugin package.""" plugin_name: str package_name: str source_description: str command: list[str] manager: str - tap_alias: str - trusted_tap: bool + catalog_alias: str + trusted_catalog: bool @dataclass(frozen=True) @@ -183,13 +222,13 @@ class InstalledPluginInfo: entry_point_value: str -def get_default_plugin_tap_url() -> str: - """Return the built-in plugin tap URL, honoring a local override for QA/staging.""" - return os.getenv(DEFAULT_PLUGIN_TAP_URL_ENV_VAR, DEFAULT_PLUGIN_TAP_URL) +def get_default_plugin_catalog_url() -> str: + """Return the built-in plugin catalog URL, honoring a local override for QA/staging.""" + return os.getenv(DEFAULT_PLUGIN_CATALOG_URL_ENV_VAR, DEFAULT_PLUGIN_CATALOG_URL) def validate_plugin_catalog_payload(payload: object, *, source: str) -> None: - """Validate a decoded plugin tap catalog against the schema v2 contract.""" + """Validate a decoded plugin catalog against the schema v2 contract.""" try: _validate_plugin_catalog_payload(payload) except PluginCatalogError as e: @@ -208,36 +247,30 @@ def _validate_plugin_catalog_payload(payload: object) -> None: f"unsupported catalog schema_version {schema_version!r}; expected {PLUGIN_CATALOG_SCHEMA_VERSION}" ) - plugins = catalog["plugins"] - if not isinstance(plugins, list): - raise PluginCatalogError("catalog document has invalid plugins; expected a list") + packages = catalog["packages"] + if not isinstance(packages, list): + raise PluginCatalogError("catalog document has invalid packages; expected a list") runtime_names: dict[str, tuple[str, str]] = {} - for index, raw_plugin in enumerate(plugins): - package_name, plugin_name, entry_point_name = _validate_catalog_plugin(raw_plugin, index) - previous = runtime_names.get(entry_point_name) - if previous is not None: - previous_package, previous_plugin_name = previous - raise PluginCatalogError( - f"duplicate runtime plugin name {entry_point_name!r} from " - f"catalog plugin {previous_plugin_name!r} in package {previous_package!r} and " - f"catalog plugin {plugin_name!r} in package {package_name!r}" - ) - runtime_names[entry_point_name] = (package_name, plugin_name) - - -def _validate_catalog_plugin(raw_plugin: object, index: int) -> tuple[str, str, str]: - context = f"catalog plugins[{index}]" - plugin = _required_catalog_object(context, raw_plugin, CATALOG_PLUGIN_KEYS) - package = _required_catalog_object(f"{context}.package", plugin["package"], CATALOG_PACKAGE_KEYS) - entry_point = _required_catalog_object( - f"{context}.entry_point", - plugin["entry_point"], - CATALOG_ENTRY_POINT_KEYS, - ) + for index, raw_package in enumerate(packages): + for package_name, plugin_name, entry_point_name in _validate_catalog_package(raw_package, index): + previous = runtime_names.get(plugin_name) + if previous is not None: + previous_package, previous_entry_point_name = previous + raise PluginCatalogError( + f"duplicate runtime plugin name {plugin_name!r} from " + f"{previous_package!r} entry point {previous_entry_point_name!r} and " + f"{package_name!r} entry point {entry_point_name!r}" + ) + runtime_names[plugin_name] = (package_name, entry_point_name) + + +def _validate_catalog_package(raw_package: object, index: int) -> list[tuple[str, str, str]]: + context = f"catalog packages[{index}]" + package = _required_catalog_object(context, raw_package, CATALOG_PACKAGE_KEYS) compatibility = _required_catalog_object( f"{context}.compatibility", - plugin["compatibility"], + package["compatibility"], CATALOG_COMPATIBILITY_KEYS, ) python_compatibility = _required_catalog_object( @@ -250,12 +283,45 @@ def _validate_catalog_plugin(raw_plugin: object, index: int) -> tuple[str, str, compatibility["data_designer"], CATALOG_DATA_DESIGNER_COMPATIBILITY_KEYS, ) - source = _required_catalog_object(f"{context}.source", plugin["source"]) - docs = _required_catalog_object(f"{context}.docs", plugin["docs"], CATALOG_DOCS_KEYS) + install = _required_catalog_object(f"{context}.install", package["install"]) + docs = _required_catalog_object(f"{context}.docs", package["docs"], CATALOG_DOCS_KEYS) - package_name = _catalog_package_name(f"{context}.package.name", package["name"]) - _catalog_version(package_name, f"{context}.package.version", package["version"]) - _validate_package_path(package_name, _required_catalog_string(f"{context}.package.path", package["path"])) + package_name = _catalog_package_name(f"{context}.name", package["name"]) + _required_catalog_string(f"{context}.description", package["description"]) + _catalog_version_specifier( + package_name, + f"{context}.compatibility.python.specifier", + python_compatibility["specifier"], + ) + _catalog_data_designer_compatibility( + package_name, + f"{context}.compatibility.data_designer", + data_designer_compatibility, + ) + _validate_install_metadata(package_name, f"{context}.install", install) + _catalog_http_url(f"{context}.docs.url", docs["url"]) + + plugins = package["plugins"] + if not isinstance(plugins, list) or not plugins: + raise PluginCatalogError(f"{context}.plugins is invalid; expected a non-empty list") + + return [ + _validate_catalog_plugin( + raw_plugin, + package_name=package_name, + context=f"{context}.plugins[{plugin_index}]", + ) + for plugin_index, raw_plugin in enumerate(plugins) + ] + + +def _validate_catalog_plugin(raw_plugin: object, *, package_name: str, context: str) -> tuple[str, str, str]: + plugin = _required_catalog_object(context, raw_plugin, CATALOG_PLUGIN_KEYS) + entry_point = _required_catalog_object( + f"{context}.entry_point", + plugin["entry_point"], + CATALOG_ENTRY_POINT_KEYS, + ) plugin_type = _required_catalog_string(f"{context}.plugin_type", plugin["plugin_type"]) if plugin_type not in SUPPORTED_PLUGIN_TYPE_VALUES: @@ -272,22 +338,40 @@ def _validate_catalog_plugin(raw_plugin: object, index: int) -> tuple[str, str, ) entry_point_name = _required_catalog_string(f"{context}.entry_point.name", entry_point["name"]) _required_catalog_string(f"{context}.entry_point.value", entry_point["value"]) - _required_catalog_string(f"{context}.description", plugin["description"]) - _catalog_version_specifier( - package_name, - f"{context}.compatibility.python.specifier", - python_compatibility["specifier"], - ) - _catalog_data_designer_compatibility( - package_name, - f"{context}.compatibility.data_designer", - data_designer_compatibility, - ) - _validate_source_metadata(package_name, source) - _catalog_http_url(f"{context}.docs.url", docs["url"]) return package_name, plugin_name, entry_point_name +def _validate_install_metadata(package_name: str, context: str, install: dict[str, object]) -> None: + keys = set(install) + missing_keys = CATALOG_INSTALL_REQUIRED_KEYS - keys + extra_keys = keys - CATALOG_INSTALL_REQUIRED_KEYS - CATALOG_INSTALL_OPTIONAL_KEYS + if missing_keys or extra_keys: + expected_required = _format_catalog_keys(CATALOG_INSTALL_REQUIRED_KEYS) + expected_optional = _format_catalog_keys(CATALOG_INSTALL_OPTIONAL_KEYS) + raise PluginCatalogError( + f"package {package_name!r} has invalid install fields; " + f"expected {{{expected_required}; optional {{{expected_optional}}}}}, " + f"got {{{_format_catalog_keys(keys)}}}" + ) + + requirement_text = _required_catalog_string(f"{context}.requirement", install["requirement"]) + try: + requirement = Requirement(requirement_text) + except InvalidRequirement as e: + raise PluginCatalogError( + f"package {package_name!r} has invalid {context}.requirement {requirement_text!r}: {e}" + ) from e + if canonicalize_name(requirement.name) != canonicalize_name(package_name): + raise PluginCatalogError( + f"package {package_name!r} has invalid {context}.requirement {requirement_text!r}; " + f"expected a requirement for {package_name!r}" + ) + + index_url = install.get("index_url") + if index_url is not None: + _catalog_http_url(f"package {package_name!r} install.index_url", index_url) + + def _required_catalog_object( context: str, value: object, @@ -332,15 +416,6 @@ def _catalog_package_name(context: str, value: object) -> str: return package_name -def _catalog_version(package_name: str, context: str, value: object) -> str: - raw_version = _required_catalog_string(context, value) - try: - Version(raw_version) - except InvalidVersion as e: - raise PluginCatalogError(f"package {package_name!r} has invalid {context} {raw_version!r}: {e}") from e - return raw_version - - def _catalog_version_specifier(package_name: str, context: str, value: object) -> str: raw_specifier = _required_catalog_string(context, value) try: @@ -387,93 +462,6 @@ def _catalog_data_designer_compatibility( ) -def _validate_source_metadata(package_name: str, source: dict[str, object]) -> None: - source_type = source.get("type") - if source_type == "pypi": - _validate_pypi_source_metadata(package_name, source) - return - if source_type == "git": - _validate_git_source_metadata(package_name, source) - return - if source_type == "path": - _validate_path_source_metadata(package_name, source) - return - raise PluginCatalogError( - f"package {package_name!r} has invalid source.type {source_type!r}; expected one of 'pypi', 'git', or 'path'" - ) - - -def _validate_pypi_source_metadata(package_name: str, source: dict[str, object]) -> None: - _validate_source_keys(package_name, source, "pypi", {"type", "package"}) - source_package = _required_source_string(package_name, source, "pypi", "package") - if source_package != package_name: - raise PluginCatalogError( - f"package {package_name!r} has invalid pypi source package {source_package!r}; " - "expected the source package to match package.name" - ) - - -def _validate_git_source_metadata(package_name: str, source: dict[str, object]) -> None: - _validate_source_keys(package_name, source, "git", {"type", "url", "ref", "subdirectory"}) - url = _required_source_string(package_name, source, "git", "url") - _required_source_string(package_name, source, "git", "ref") - subdirectory = _required_source_string(package_name, source, "git", "subdirectory") - _validate_package_path(package_name, subdirectory) - parsed = urlparse(url) - if parsed.scheme not in {"http", "https"} or not parsed.netloc: - raise PluginCatalogError( - f"package {package_name!r} has invalid git source url {url!r}; expected an absolute HTTP(S) URL" - ) - - -def _validate_path_source_metadata(package_name: str, source: dict[str, object]) -> None: - _validate_source_keys(package_name, source, "path", {"type", "path", "editable"}) - path = _required_source_string(package_name, source, "path", "path") - _validate_package_path(package_name, path) - editable = source.get("editable") - if not isinstance(editable, bool): - raise PluginCatalogError(f"package {package_name!r} has invalid path source field 'editable'; expected a bool") - - -def _validate_source_keys( - package_name: str, - source: dict[str, object], - source_type: str, - expected_keys: set[str], -) -> None: - keys = set(source) - if keys != expected_keys: - raise PluginCatalogError( - f"package {package_name!r} has invalid {source_type!r} source fields; " - f"expected {{{_format_catalog_keys(expected_keys)}}}, got {{{_format_catalog_keys(keys)}}}" - ) - - -def _required_source_string(package_name: str, source: dict[str, object], source_type: str, key: str) -> str: - value = source.get(key) - if not isinstance(value, str) or not value: - raise PluginCatalogError( - f"package {package_name!r} has invalid {source_type!r} source field {key!r}; expected a non-empty string" - ) - return value - - -def _validate_package_path(package_name: str, value: str) -> None: - parts = value.split("/") - if ( - "\\" in value - or value.startswith("/") - or len(parts) < 2 - or parts[0] != PACKAGE_PATH_ROOT - or any(part in {"", ".", ".."} for part in parts) - or not all(PACKAGE_PATH_SEGMENT_PATTERN.fullmatch(part) for part in parts[1:]) - ): - raise PluginCatalogError( - f"package {package_name!r} has invalid package path {value!r}; " - f"expected a normalized repository-relative path under {PACKAGE_PATH_ROOT!r}" - ) - - def _catalog_http_url(context: str, value: object) -> str: url = _required_catalog_string(context, value) parsed = urlparse(url) diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py similarity index 58% rename from packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py rename to packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py index 13fac8fce..5bb87e4dd 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_tap_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py @@ -14,140 +14,144 @@ from pydantic import ValidationError from data_designer.cli.plugin_catalog import ( - DEFAULT_PLUGIN_TAP_ALIAS, + DEFAULT_PLUGIN_CATALOG_ALIAS, MAX_PLUGIN_CATALOG_SIZE_BYTES, - PLUGIN_TAP_CACHE_DIR_NAME, - PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, - PLUGIN_TAPS_FILE_NAME, + PLUGIN_CATALOG_CACHE_DIR_NAME, + PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, + PLUGIN_CATALOGS_FILE_NAME, PluginCatalog, + PluginCatalogConfig, PluginCatalogError, - PluginTapConfig, - PluginTapRegistry, - get_default_plugin_tap_url, + PluginCatalogRegistry, + get_default_plugin_catalog_url, validate_plugin_catalog_payload, ) from data_designer.cli.repositories.base import ConfigRepository from data_designer.config.utils.io_helpers import load_config_file, save_config_file -class PluginTapRepository(ConfigRepository[PluginTapRegistry]): - """Repository for plugin tap aliases and cached catalog payloads.""" +class PluginCatalogRepository(ConfigRepository[PluginCatalogRegistry]): + """Repository for plugin catalog aliases and cached catalog payloads.""" @property def config_file(self) -> Path: - """Get the plugin tap configuration file path.""" - return self.config_dir / PLUGIN_TAPS_FILE_NAME + """Get the plugin catalog configuration file path.""" + return self.config_dir / PLUGIN_CATALOGS_FILE_NAME @property def cache_dir(self) -> Path: - """Get the plugin tap cache directory path.""" - return self.config_dir / PLUGIN_TAP_CACHE_DIR_NAME + """Get the plugin catalog cache directory path.""" + return self.config_dir / PLUGIN_CATALOG_CACHE_DIR_NAME - def load(self) -> PluginTapRegistry | None: - """Load user-configured plugin taps.""" + def load(self) -> PluginCatalogRegistry | None: + """Load user-configured plugin catalogs.""" if not self.exists(): return None try: config_dict = load_config_file(self.config_file) - return PluginTapRegistry.model_validate(config_dict) + return PluginCatalogRegistry.model_validate(config_dict) except Exception: return None - def save(self, config: PluginTapRegistry) -> None: - """Save user-configured plugin taps.""" + def save(self, config: PluginCatalogRegistry) -> None: + """Save user-configured plugin catalogs.""" config_dict = config.model_dump(mode="json", exclude_none=True) save_config_file(self.config_file, config_dict) - def list_taps(self) -> list[PluginTapConfig]: - """Return the built-in NVIDIA tap followed by user-configured taps.""" - taps = [self.default_tap()] + def list_catalogs(self) -> list[PluginCatalogConfig]: + """Return the built-in NVIDIA catalog followed by user-configured catalogs.""" + catalogs = [self.default_catalog()] registry = self.load() if registry is not None: - taps.extend(sorted(registry.taps, key=lambda tap: tap.alias.casefold())) - return taps + catalogs.extend(sorted(registry.catalogs, key=lambda catalog: catalog.alias.casefold())) + return catalogs - def get_tap(self, alias: str | None = None) -> PluginTapConfig | None: - """Return a tap by alias, defaulting to the built-in NVIDIA tap.""" - resolved_alias = alias or DEFAULT_PLUGIN_TAP_ALIAS - return next((tap for tap in self.list_taps() if _same_alias(tap.alias, resolved_alias)), None) + def get_catalog(self, alias: str | None = None) -> PluginCatalogConfig | None: + """Return a catalog by alias, defaulting to the built-in NVIDIA catalog.""" + resolved_alias = alias or DEFAULT_PLUGIN_CATALOG_ALIAS + return next((catalog for catalog in self.list_catalogs() if _same_alias(catalog.alias, resolved_alias)), None) - def add_tap( + def add_catalog( self, alias: str, url: str, *, trusted: bool = False, - cache_ttl_seconds: int = PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, - ) -> PluginTapConfig: - """Persist a new tap alias. + cache_ttl_seconds: int = PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, + ) -> PluginCatalogConfig: + """Persist a new catalog alias. Raises: - ValueError: If the alias already exists or is reserved for the built-in tap. + ValueError: If the alias already exists or is reserved for the built-in catalog. """ - if self.get_tap(alias) is not None: - raise ValueError(f"Plugin tap alias {alias!r} already exists") + if self.get_catalog(alias) is not None: + raise ValueError(f"Plugin catalog alias {alias!r} already exists") - tap = PluginTapConfig( + catalog = PluginCatalogConfig( alias=alias, - url=normalize_tap_location(url), + url=normalize_catalog_location(url), trusted=trusted, cache_ttl_seconds=cache_ttl_seconds, ) - registry = self.load() or PluginTapRegistry() - registry.taps.append(tap) - registry.taps = sorted(registry.taps, key=lambda item: item.alias.casefold()) + registry = self.load() or PluginCatalogRegistry() + registry.catalogs.append(catalog) + registry.catalogs = sorted(registry.catalogs, key=lambda item: item.alias.casefold()) self.save(registry) - return tap + return catalog - def remove_tap(self, alias: str) -> None: - """Remove a user-configured tap alias. + def remove_catalog(self, alias: str) -> None: + """Remove a user-configured catalog alias. Raises: ValueError: If the alias is reserved or does not exist. """ - if _same_alias(alias, DEFAULT_PLUGIN_TAP_ALIAS): - raise ValueError(f"Cannot remove the built-in {DEFAULT_PLUGIN_TAP_ALIAS!r} plugin tap") + if _same_alias(alias, DEFAULT_PLUGIN_CATALOG_ALIAS): + raise ValueError(f"Cannot remove the built-in {DEFAULT_PLUGIN_CATALOG_ALIAS!r} plugin catalog") registry = self.load() - matching_tap = next((tap for tap in registry.taps if _same_alias(tap.alias, alias)), None) if registry else None - if registry is None or matching_tap is None: - raise ValueError(f"Plugin tap alias {alias!r} not found") + matching_catalog = ( + next((catalog for catalog in registry.catalogs if _same_alias(catalog.alias, alias)), None) + if registry + else None + ) + if registry is None or matching_catalog is None: + raise ValueError(f"Plugin catalog alias {alias!r} not found") - registry.taps = [tap for tap in registry.taps if not _same_alias(tap.alias, alias)] - if registry.taps: + registry.catalogs = [catalog for catalog in registry.catalogs if not _same_alias(catalog.alias, alias)] + if registry.catalogs: self.save(registry) else: self.delete() - self._remove_cache_files(matching_tap) + self._remove_cache_files(matching_catalog) def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> PluginCatalog: - """Load a tap catalog from cache or source.""" - tap = self.get_tap(alias) - if tap is None: - raise ValueError(f"Plugin tap alias {alias!r} not found") + """Load a catalog from cache or source.""" + catalog_config = self.get_catalog(alias) + if catalog_config is None: + raise ValueError(f"Plugin catalog alias {alias!r} not found") if not refresh: - cached_catalog = self._load_cached_catalog(tap, require_fresh=True) + cached_catalog = self._load_cached_catalog(catalog_config, require_fresh=True) if cached_catalog is not None: return cached_catalog try: - payload = self._fetch_catalog_payload(tap.url) - catalog = self._validate_catalog(payload, source=tap.url) + payload = self._fetch_catalog_payload(catalog_config.url) + catalog = self._validate_catalog(payload, source=catalog_config.url) except Exception: if not refresh: - cached_catalog = self._load_cached_catalog(tap, require_fresh=False) + cached_catalog = self._load_cached_catalog(catalog_config, require_fresh=False) if cached_catalog is not None: return cached_catalog raise - self._save_catalog_cache(tap, payload) + self._save_catalog_cache(catalog_config, payload) return catalog - def _load_cached_catalog(self, tap: PluginTapConfig, *, require_fresh: bool) -> PluginCatalog | None: - cache_file = self._cache_file(tap) + def _load_cached_catalog(self, catalog: PluginCatalogConfig, *, require_fresh: bool) -> PluginCatalog | None: + cache_file = self._cache_file(catalog) if not cache_file.exists(): return None @@ -157,38 +161,38 @@ def _load_cached_catalog(self, tap: PluginTapConfig, *, require_fresh: bool) -> fetched_at = datetime.fromisoformat(cache_payload["fetched_at"]) if fetched_at.tzinfo is None: fetched_at = fetched_at.replace(tzinfo=timezone.utc) - if require_fresh and tap.cache_ttl_seconds == 0: + if require_fresh and catalog.cache_ttl_seconds == 0: return None if require_fresh: age_seconds = (datetime.now(timezone.utc) - fetched_at).total_seconds() - if age_seconds > tap.cache_ttl_seconds: + if age_seconds > catalog.cache_ttl_seconds: return None catalog_payload = cache_payload["catalog"] return self._validate_catalog(catalog_payload, source=str(cache_file)) except Exception: return None - def _save_catalog_cache(self, tap: PluginTapConfig, catalog_payload: dict[str, object]) -> None: + def _save_catalog_cache(self, catalog: PluginCatalogConfig, catalog_payload: dict[str, object]) -> None: self.cache_dir.mkdir(parents=True, exist_ok=True) cache_payload = { - "tap_alias": tap.alias, - "tap_url": tap.url, + "catalog_alias": catalog.alias, + "catalog_url": catalog.url, "fetched_at": datetime.now(timezone.utc).isoformat(), "catalog": catalog_payload, } - with open(self._cache_file(tap), "w") as f: + with open(self._cache_file(catalog), "w") as f: json.dump(cache_payload, f, indent=2, sort_keys=True) - def _cache_file(self, tap: PluginTapConfig) -> Path: - url_hash = hashlib.sha256(tap.url.encode("utf-8")).hexdigest()[:12] - return self.cache_dir / f"{tap.alias}-{url_hash}.json" + def _cache_file(self, catalog: PluginCatalogConfig) -> Path: + url_hash = hashlib.sha256(catalog.url.encode("utf-8")).hexdigest()[:12] + return self.cache_dir / f"{catalog.alias}-{url_hash}.json" - def _remove_cache_files(self, tap: PluginTapConfig) -> None: + def _remove_cache_files(self, catalog: PluginCatalogConfig) -> None: if not self.cache_dir.exists(): return - self._cache_file(tap).unlink(missing_ok=True) - legacy_cache_file = self.cache_dir / f"{tap.alias}.json" + self._cache_file(catalog).unlink(missing_ok=True) + legacy_cache_file = self.cache_dir / f"{catalog.alias}.json" legacy_cache_file.unlink(missing_ok=True) for cache_file in self.cache_dir.glob("*.json"): @@ -197,8 +201,8 @@ def _remove_cache_files(self, tap: PluginTapConfig) -> None: cache_payload = json.load(f) except Exception: continue - cached_alias = cache_payload.get("tap_alias") - if isinstance(cached_alias, str) and _same_alias(cached_alias, tap.alias): + cached_alias = cache_payload.get("catalog_alias") + if isinstance(cached_alias, str) and _same_alias(cached_alias, catalog.alias): cache_file.unlink(missing_ok=True) @staticmethod @@ -217,20 +221,20 @@ def _validate_catalog(payload: dict, *, source: str) -> PluginCatalog: return catalog @staticmethod - def default_tap() -> PluginTapConfig: - """Return the built-in NVIDIA plugin tap configuration.""" - return PluginTapConfig( - alias=DEFAULT_PLUGIN_TAP_ALIAS, - url=get_default_plugin_tap_url(), + def default_catalog() -> PluginCatalogConfig: + """Return the built-in NVIDIA plugin catalog configuration.""" + return PluginCatalogConfig( + alias=DEFAULT_PLUGIN_CATALOG_ALIAS, + url=get_default_plugin_catalog_url(), trusted=True, - cache_ttl_seconds=PLUGIN_TAP_DEFAULT_CACHE_TTL_SECONDS, + cache_ttl_seconds=PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, ) -def normalize_tap_location(location: str) -> str: - """Normalize a tap repository, catalog URL, or local path to a catalog location.""" +def normalize_catalog_location(location: str) -> str: + """Normalize a catalog repository, catalog URL, or local path to a catalog location.""" if _is_http_url(location): - return _normalize_tap_url(location) + return _normalize_catalog_url(location) path = Path(location).expanduser() if path.suffix.lower() == ".json": @@ -242,7 +246,7 @@ def _same_alias(left: str, right: str) -> bool: return left.casefold() == right.casefold() -def _normalize_tap_url(url: str) -> str: +def _normalize_catalog_url(url: str) -> str: parsed = urlparse(url) hostname = parsed.hostname or "" segments = [segment for segment in parsed.path.split("/") if segment] @@ -257,8 +261,8 @@ def _normalize_tap_url(url: str) -> str: return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}" if len(segments) >= 4 and segments[2] == "tree": ref = segments[3] - tap_root = "/".join(segments[4:]) - catalog_path = f"{tap_root}/catalog/plugins.json" if tap_root else "catalog/plugins.json" + catalog_root = "/".join(segments[4:]) + catalog_path = f"{catalog_root}/catalog/plugins.json" if catalog_root else "catalog/plugins.json" return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{catalog_path}" return url diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index b7aa1156e..e7cd0ffc1 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -14,14 +14,15 @@ from packaging.version import InvalidVersion, Version from data_designer.cli.plugin_catalog import ( - DEFAULT_PLUGIN_TAP_ALIAS, + DEFAULT_PLUGIN_CATALOG_ALIAS, + PLUGIN_ENTRY_POINT_GROUP, CompatibilityResult, InstalledPluginInfo, + PluginCatalogConfig, PluginCatalogEntry, PluginCompatibilityTarget, - PluginTapConfig, ) -from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository +from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository class PluginCatalogService: @@ -29,7 +30,7 @@ class PluginCatalogService: def __init__( self, - repository: PluginTapRepository, + repository: PluginCatalogRepository, *, python_version: str | None = None, data_designer_version: str | None = None, @@ -40,14 +41,14 @@ def __init__( def list_entries( self, - tap_alias: str | None = None, + catalog_alias: str | None = None, *, refresh: bool = False, include_incompatible: bool = False, ) -> list[PluginCatalogEntry]: - """List catalog entries for a tap, filtering incompatible entries by default.""" - catalog = self.repository.load_catalog(tap_alias, refresh=refresh) - entries = sorted(catalog.plugins, key=lambda entry: (entry.name, _entry_version_sort_key(entry))) + """List catalog entries for a catalog, filtering incompatible entries by default.""" + catalog = self.repository.load_catalog(catalog_alias, refresh=refresh) + entries = sorted(catalog.entries, key=lambda entry: (canonicalize_name(entry.package.name), entry.name)) if include_incompatible: return entries return [entry for entry in entries if self.evaluate_compatibility(entry).is_compatible] @@ -55,7 +56,7 @@ def list_entries( def search_entries( self, query: str, - tap_alias: str | None = None, + catalog_alias: str | None = None, *, refresh: bool = False, include_incompatible: bool = False, @@ -68,7 +69,7 @@ def search_entries( return [ entry for entry in self.list_entries( - tap_alias, + catalog_alias, refresh=refresh, include_incompatible=include_incompatible, ) @@ -78,26 +79,47 @@ def search_entries( def get_entry( self, name: str, - tap_alias: str | None = None, + catalog_alias: str | None = None, *, refresh: bool = False, include_incompatible: bool = True, ) -> PluginCatalogEntry: - """Return the newest catalog entry by plugin name.""" - entries = self.list_entries(tap_alias, refresh=refresh, include_incompatible=True) - matches = [entry for entry in entries if entry.name == name] + """Return a catalog entry by runtime plugin name or package name.""" + entries = self.list_entries(catalog_alias, refresh=refresh, include_incompatible=True) + canonical_name = canonicalize_name(name) + matches = [ + entry for entry in entries if entry.name == name or canonicalize_name(entry.package.name) == canonical_name + ] compatible_matches = [entry for entry in matches if self.evaluate_compatibility(entry).is_compatible] if compatible_matches: - return max(compatible_matches, key=_entry_version_sort_key) + return sorted(compatible_matches, key=lambda entry: (canonicalize_name(entry.package.name), entry.name))[0] if matches and include_incompatible: - return max(matches, key=_entry_version_sort_key) + return sorted(matches, key=lambda entry: (canonicalize_name(entry.package.name), entry.name))[0] - resolved_alias = tap_alias or DEFAULT_PLUGIN_TAP_ALIAS + resolved_alias = catalog_alias or DEFAULT_PLUGIN_CATALOG_ALIAS if matches: - raise ValueError( - f"Plugin {name!r} was found in tap {resolved_alias!r}, but no compatible version is available" + raise ValueError(f"Plugin package {name!r} was found in catalog {resolved_alias!r}, but is not compatible") + raise ValueError(f"Plugin or package {name!r} was not found in catalog {resolved_alias!r}") + + def get_package_entries( + self, + package_name: str, + catalog_alias: str | None = None, + *, + refresh: bool = False, + include_incompatible: bool = True, + ) -> list[PluginCatalogEntry]: + """Return all runtime plugin entries declared by one catalog package.""" + canonical_package_name = canonicalize_name(package_name) + return [ + entry + for entry in self.list_entries( + catalog_alias, + refresh=refresh, + include_incompatible=include_incompatible, ) - raise ValueError(f"Plugin {name!r} was not found in tap {resolved_alias!r}") + if canonicalize_name(entry.package.name) == canonical_package_name + ] @staticmethod def group_entries_by_package(entries: Iterable[PluginCatalogEntry]) -> dict[str, list[PluginCatalogEntry]]: @@ -134,40 +156,40 @@ def evaluate_compatibility(self, entry: PluginCatalogEntry) -> CompatibilityResu ) return CompatibilityResult(is_compatible=not reasons, reasons=reasons) - def list_taps(self) -> list[PluginTapConfig]: - """List available plugin taps.""" - return self.repository.list_taps() + def list_catalogs(self) -> list[PluginCatalogConfig]: + """List available plugin catalogs.""" + return self.repository.list_catalogs() - def get_tap(self, alias: str | None = None) -> PluginTapConfig: - """Return a plugin tap or raise a user-facing error.""" - tap = self.repository.get_tap(alias) - if tap is None: - raise ValueError(f"Plugin tap alias {alias!r} not found") - return tap + def get_catalog(self, alias: str | None = None) -> PluginCatalogConfig: + """Return a plugin catalog or raise a user-facing error.""" + catalog = self.repository.get_catalog(alias) + if catalog is None: + raise ValueError(f"Plugin catalog alias {alias!r} not found") + return catalog - def add_tap( + def add_catalog( self, alias: str, url: str, *, trusted: bool, cache_ttl_seconds: int, - ) -> PluginTapConfig: - """Add a plugin tap alias.""" - return self.repository.add_tap( + ) -> PluginCatalogConfig: + """Add a plugin catalog alias.""" + return self.repository.add_catalog( alias, url, trusted=trusted, cache_ttl_seconds=cache_ttl_seconds, ) - def remove_tap(self, alias: str) -> None: - """Remove a plugin tap alias.""" - self.repository.remove_tap(alias) + def remove_catalog(self, alias: str) -> None: + """Remove a plugin catalog alias.""" + self.repository.remove_catalog(alias) def list_installed_plugins(self) -> list[InstalledPluginInfo]: """List installed Data Designer plugin entry points without importing plugin modules.""" - entry_points = importlib.metadata.entry_points(group="data_designer.plugins") + entry_points = importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP) installed_plugins = [ InstalledPluginInfo(name=entry_point.name, entry_point_value=entry_point.value) for entry_point in entry_points @@ -226,25 +248,15 @@ def _entry_search_text(entry: PluginCatalogEntry) -> str: entry.plugin_type.value, entry.description, entry.package.name, - entry.package.version or "", - entry.package.path or "", + entry.install.requirement, + entry.install.index_url or "", entry.entry_point.name, entry.entry_point.value, - entry.source.type if entry.source is not None else "", - entry.source.package if entry.source is not None and entry.source.package else "", - entry.source.url if entry.source is not None and entry.source.url else "", entry.docs.url if entry.docs is not None and entry.docs.url else "", ] return " ".join(values).lower() -def _entry_version_sort_key(entry: PluginCatalogEntry) -> Version: - try: - return Version(entry.package.version or "0") - except InvalidVersion: - return Version("0") - - def _major_minor(version: str) -> str: parts = version.split(".") if len(parts) < 2: diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 0c8aa511c..ac2eada53 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -9,15 +9,13 @@ import subprocess import sys from collections.abc import Callable -from pathlib import Path -from urllib.parse import urlparse from data_designer.cli.plugin_catalog import ( PLUGIN_ENTRY_POINT_GROUP, + PYPI_SIMPLE_INDEX_URL, InstallPlan, + PluginCatalogConfig, PluginCatalogEntry, - PluginSourceInfo, - PluginTapConfig, ) InstallRunner = Callable[[list[str]], int] @@ -32,13 +30,13 @@ def __init__(self, runner: InstallRunner | None = None) -> None: def build_install_plan( self, entry: PluginCatalogEntry, - tap: PluginTapConfig, + catalog: PluginCatalogConfig, *, manager: str = "auto", ) -> InstallPlan: """Build the exact package-manager command for one catalog entry.""" resolved_manager = _resolve_manager(manager) - install_args, source_description = _install_args_for_entry(entry, tap) + install_args, source_description = _install_args_for_entry(entry, resolved_manager) command = _base_command(resolved_manager) + install_args return InstallPlan( plugin_name=entry.name, @@ -46,8 +44,8 @@ def build_install_plan( source_description=source_description, command=command, manager=resolved_manager, - tap_alias=tap.alias, - trusted_tap=tap.trusted, + catalog_alias=catalog.alias, + trusted_catalog=catalog.trusted, ) def install(self, plan: InstallPlan) -> None: @@ -62,11 +60,15 @@ def install(self, plan: InstallPlan) -> None: def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: """Verify the plugin's declared entry point is installed.""" + return self.verify_entry_points([entry]) + + def verify_entry_points(self, entries: list[PluginCatalogEntry]) -> bool: + """Verify every declared entry point for an installed catalog package.""" importlib.invalidate_caches() - return any( - entry_point.name == entry.entry_point.name - for entry_point in importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP) - ) + installed_entry_point_names = { + entry_point.name for entry_point in importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP) + } + return bool(entries) and all(entry.entry_point.name in installed_entry_point_names for entry in entries) def _run_subprocess(command: list[str]) -> int: @@ -90,65 +92,18 @@ def _base_command(manager: str) -> list[str]: return [sys.executable, "-m", "pip", "install"] -def _install_args_for_entry(entry: PluginCatalogEntry, tap: PluginTapConfig) -> tuple[list[str], str]: - source = entry.source - if source is None: - raise ValueError( - f"Plugin {entry.name!r} cannot be installed because the catalog entry does not declare a source" - ) - - source_type = source.type.lower() - if source_type == "pypi": - target = _pypi_target(entry, source) - return [target], target - if source_type == "git": - target = _git_target(entry, source) - return [target], target - if source_type == "path": - args = _path_args(entry, source, tap) - return args, " ".join(args) - - raise ValueError(f"Plugin {entry.name!r} declares unsupported install source type {source.type!r}") - - -def _pypi_target(entry: PluginCatalogEntry, source: PluginSourceInfo) -> str: - package_name = source.package or entry.package.name - if entry.package.version: - return f"{package_name}=={entry.package.version}" - return package_name - - -def _git_target(entry: PluginCatalogEntry, source: PluginSourceInfo) -> str: - url = _required(source.url, "url", "git") - ref = _required(source.ref, "ref", "git") - subdirectory = _required(source.subdirectory, "subdirectory", "git") - return f"{entry.package.name} @ git+{url}@{ref}#subdirectory={subdirectory}" +def _install_args_for_entry(entry: PluginCatalogEntry, manager: str) -> tuple[list[str], str]: + requirement = entry.install.requirement + index_url = entry.install.index_url + if index_url is None: + return [requirement], requirement - -def _path_args(entry: PluginCatalogEntry, source: PluginSourceInfo, tap: PluginTapConfig) -> list[str]: - path = _required(source.path, "path", "path") - normalized_path = str(_resolve_path_source(path, tap)) - if source.editable: - return ["-e", normalized_path] - return [normalized_path] - - -def _required(value: str | None, field_name: str, source_type: str) -> str: - if value is None: - raise ValueError(f"Plugin install source type {source_type!r} requires {field_name!r}") - return value - - -def _resolve_path_source(path: str, tap: PluginTapConfig) -> Path: - source_path = Path(path).expanduser() - if source_path.is_absolute(): - return source_path - - tap_location = urlparse(tap.url) - if tap_location.scheme in {"http", "https"}: - raise ValueError("Relative path plugin sources require a local plugin tap") - - tap_catalog_path = Path(tap.url).expanduser() - if tap_catalog_path.name == "plugins.json" and tap_catalog_path.parent.name == "catalog": - return tap_catalog_path.parent.parent / source_path - return tap_catalog_path.parent / source_path + if manager == "uv": + return ( + ["--default-index", PYPI_SIMPLE_INDEX_URL, "--index", index_url, requirement], + f"{requirement} via {index_url}", + ) + return ( + ["--extra-index-url", index_url, requirement], + f"{requirement} via {index_url}", + ) diff --git a/packages/data-designer/tests/cli/commands/test_plugins_command.py b/packages/data-designer/tests/cli/commands/test_plugins_command.py index c74f0e9e2..4605ce423 100644 --- a/packages/data-designer/tests/cli/commands/test_plugins_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugins_command.py @@ -17,11 +17,11 @@ def test_plugins_list_command_delegates_to_controller(mock_ctrl_cls: MagicMock) mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "--tap", "research", "list", "--refresh", "--include-incompatible"]) + result = runner.invoke(app, ["plugins", "--catalog", "research", "list", "--refresh", "--include-incompatible"]) assert result.exit_code == 0 mock_ctrl.run_list.assert_called_once_with( - tap_alias="research", + catalog_alias="research", refresh=True, include_incompatible=True, ) @@ -32,12 +32,12 @@ def test_plugins_search_command_delegates_to_controller(mock_ctrl_cls: MagicMock mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "search", "github", "--tap", "research"]) + result = runner.invoke(app, ["plugins", "search", "github", "--catalog", "research"]) assert result.exit_code == 0 mock_ctrl.run_search.assert_called_once_with( "github", - tap_alias="research", + catalog_alias="research", refresh=False, include_incompatible=False, ) @@ -53,7 +53,7 @@ def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMoc assert result.exit_code == 0 mock_ctrl.run_install.assert_called_once_with( "text-transform", - tap_alias=None, + catalog_alias=None, refresh=False, manager="pip", yes=True, @@ -63,7 +63,7 @@ def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMoc @patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_taps_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +def test_plugins_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl @@ -71,7 +71,7 @@ def test_plugins_taps_add_command_delegates_to_controller(mock_ctrl_cls: MagicMo app, [ "plugins", - "taps", + "catalogs", "add", "research", "https://github.com/acme/dd-plugins", @@ -82,7 +82,7 @@ def test_plugins_taps_add_command_delegates_to_controller(mock_ctrl_cls: MagicMo ) assert result.exit_code == 0 - mock_ctrl.run_taps_add.assert_called_once_with( + mock_ctrl.run_catalogs_add.assert_called_once_with( alias="research", url="https://github.com/acme/dd-plugins", trusted=True, @@ -92,35 +92,35 @@ def test_plugins_taps_add_command_delegates_to_controller(mock_ctrl_cls: MagicMo @patch("data_designer.cli.commands.plugins.print_info") @patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_installed_warns_when_parent_tap_is_unused( +def test_plugins_installed_warns_when_parent_catalog_is_unused( mock_ctrl_cls: MagicMock, mock_print_info: MagicMock, ) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "--tap", "research", "installed"]) + result = runner.invoke(app, ["plugins", "--catalog", "research", "installed"]) assert result.exit_code == 0 mock_print_info.assert_called_once_with( - "Ignoring --tap 'research'; installed plugins are discovered from the current Python environment." + "Ignoring --catalog 'research'; installed plugins are discovered from the current Python environment." ) mock_ctrl.run_installed.assert_called_once_with() @patch("data_designer.cli.commands.plugins.print_info") @patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_taps_list_warns_when_parent_tap_is_unused( +def test_plugins_catalogs_list_warns_when_parent_catalog_is_unused( mock_ctrl_cls: MagicMock, mock_print_info: MagicMock, ) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "--tap", "research", "taps", "list"]) + result = runner.invoke(app, ["plugins", "--catalog", "research", "catalogs", "list"]) assert result.exit_code == 0 mock_print_info.assert_called_once_with( - "Ignoring --tap 'research'; tap management commands operate on aliases directly." + "Ignoring --catalog 'research'; catalog management commands operate on aliases directly." ) - mock_ctrl.run_taps_list.assert_called_once_with() + mock_ctrl.run_catalogs_list.assert_called_once_with() diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index ed7be8f91..36a96ad94 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -13,8 +13,8 @@ from data_designer.cli.plugin_catalog import ( CompatibilityResult, InstallPlan, + PluginCatalogConfig, PluginCatalogEntry, - PluginTapConfig, ) @@ -34,14 +34,15 @@ def test_run_install_dry_run_renders_plan_without_installing( controller: PluginCatalogController, ) -> None: entry = _entry() - tap = _tap(trusted=True) - plan = _plan(tap) - controller.catalog_service.get_tap.return_value = tap + catalog = _catalog(trusted=True) + plan = _plan(catalog) + controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = plan - controller.run_install("text-transform", tap_alias="local", dry_run=True) + controller.run_install("text-transform", catalog_alias="local", dry_run=True) controller.catalog_service.get_entry.assert_called_once_with( "text-transform", @@ -50,7 +51,7 @@ def test_run_install_dry_run_renders_plan_without_installing( include_incompatible=True, ) controller.install_service.install.assert_not_called() - controller.install_service.verify_entry_point.assert_not_called() + controller.install_service.verify_entry_points.assert_not_called() mock_print_info.assert_any_call("Dry run complete; no changes made") assert mock_console.print.call_count >= 1 @@ -63,16 +64,17 @@ def test_run_install_blocks_incompatible_plugin_without_force( controller: PluginCatalogController, ) -> None: entry = _entry() - tap = _tap(trusted=True) - controller.catalog_service.get_tap.return_value = tap + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( False, ["Data Designer 0.5.7 does not satisfy >=99.0"], ) with pytest.raises(typer.Exit) as exc_info: - controller.run_install("text-transform", tap_alias="local") + controller.run_install("text-transform", catalog_alias="local") assert exc_info.value.exit_code == 1 controller.catalog_service.get_entry.assert_called_once_with( @@ -96,16 +98,17 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( controller: PluginCatalogController, ) -> None: entry = _entry() - tap = _tap(trusted=True) - controller.catalog_service.get_tap.return_value = tap + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( False, ["Data Designer 0.5.7 does not satisfy >=99.0"], ) - controller.install_service.build_install_plan.return_value = _plan(tap) + controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_install("text-transform", tap_alias="local", dry_run=True, force=True) + controller.run_install("text-transform", catalog_alias="local", dry_run=True, force=True) controller.catalog_service.get_entry.assert_called_once_with( "text-transform", @@ -113,7 +116,7 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( refresh=False, include_incompatible=True, ) - controller.install_service.build_install_plan.assert_called_once_with(entry, tap, manager="auto") + controller.install_service.build_install_plan.assert_called_once_with(entry, catalog, manager="auto") controller.install_service.install.assert_not_called() mock_print_error.assert_not_called() mock_print_info.assert_any_call("Dry run complete; no changes made") @@ -122,22 +125,23 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") -def test_run_install_warns_for_untrusted_tap( +def test_run_install_warns_for_untrusted_catalog( mock_print_warning: MagicMock, mock_console: MagicMock, controller: PluginCatalogController, ) -> None: entry = _entry() - tap = _tap(trusted=False) - controller.catalog_service.get_tap.return_value = tap + catalog = _catalog(trusted=False) + controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) - controller.install_service.build_install_plan.return_value = _plan(tap) + controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_install("text-transform", tap_alias="local", dry_run=True) + controller.run_install("text-transform", catalog_alias="local", dry_run=True) mock_print_warning.assert_called_once_with( - "This tap is not marked trusted. Plugin installation executes Python package code from the source above." + "This catalog is not marked trusted. Plugin installation executes Python package code from the requirement above." ) assert mock_console.print.call_count >= 1 @@ -150,19 +154,20 @@ def test_run_install_reports_success_when_verification_finds_entry_point( controller: PluginCatalogController, ) -> None: entry = _entry() - tap = _tap(trusted=True) - plan = _plan(tap) - controller.catalog_service.get_tap.return_value = tap + catalog = _catalog(trusted=True) + plan = _plan(catalog) + controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = plan - controller.install_service.verify_entry_point.return_value = True + controller.install_service.verify_entry_points.return_value = True - controller.run_install("text-transform", tap_alias="local", yes=True) + controller.run_install("text-transform", catalog_alias="local", yes=True) controller.install_service.install.assert_called_once_with(plan) - controller.install_service.verify_entry_point.assert_called_once_with(entry) - mock_print_success.assert_called_once_with("Plugin 'text-transform' installed and discovered") + controller.install_service.verify_entry_points.assert_called_once_with([entry]) + mock_print_success.assert_called_once_with("Plugin package 'data-designer-text-transform' installed and discovered") assert mock_console.print.call_count >= 1 @@ -174,34 +179,36 @@ def test_run_install_warns_when_verification_misses_entry_point( controller: PluginCatalogController, ) -> None: entry = _entry() - tap = _tap(trusted=True) - plan = _plan(tap) - controller.catalog_service.get_tap.return_value = tap + catalog = _catalog(trusted=True) + plan = _plan(catalog) + controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = plan - controller.install_service.verify_entry_point.return_value = False + controller.install_service.verify_entry_points.return_value = False - controller.run_install("text-transform", tap_alias="local", yes=True) + controller.run_install("text-transform", catalog_alias="local", yes=True) controller.install_service.install.assert_called_once_with(plan) - controller.install_service.verify_entry_point.assert_called_once_with(entry) + controller.install_service.verify_entry_points.assert_called_once_with([entry]) mock_print_warning.assert_called_once_with( - "Plugin 'text-transform' was installed, but Data Designer did not discover its entry point. " + "Plugin package 'data-designer-text-transform' was installed, but Data Designer did not discover every " + "declared entry point. " "Restart the shell or check the package entry point metadata." ) assert mock_console.print.call_count >= 1 @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") -def test_run_taps_add_wraps_invalid_alias_validation_error( +def test_run_catalogs_add_wraps_invalid_alias_validation_error( mock_print_error: MagicMock, tmp_path: Path, ) -> None: plugin_controller = PluginCatalogController(tmp_path) with pytest.raises(typer.Exit) as exc_info: - plugin_controller.run_taps_add( + plugin_controller.run_catalogs_add( alias="foo/bar", url="https://github.com/acme/dd-plugins", trusted=False, @@ -209,26 +216,26 @@ def test_run_taps_add_wraps_invalid_alias_validation_error( ) assert exc_info.value.exit_code == 1 - mock_print_error.assert_called_once_with("Invalid tap alias 'foo/bar': must match `^[A-Za-z0-9_.-]+$`") + mock_print_error.assert_called_once_with("Invalid catalog alias 'foo/bar': must match `^[A-Za-z0-9_.-]+$`") -def _tap(*, trusted: bool) -> PluginTapConfig: - return PluginTapConfig( +def _catalog(*, trusted: bool) -> PluginCatalogConfig: + return PluginCatalogConfig( alias="local", url="https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json", trusted=trusted, ) -def _plan(tap: PluginTapConfig) -> InstallPlan: +def _plan(catalog: PluginCatalogConfig) -> InstallPlan: return InstallPlan( plugin_name="text-transform", package_name="data-designer-text-transform", - source_description="data-designer-text-transform==0.1.0", - command=["python", "-m", "pip", "install", "data-designer-text-transform==0.1.0"], + source_description="data-designer-text-transform", + command=["python", "-m", "pip", "install", "data-designer-text-transform"], manager="pip", - tap_alias=tap.alias, - trusted_tap=tap.trusted, + catalog_alias=catalog.alias, + trusted_catalog=catalog.trusted, ) @@ -240,8 +247,10 @@ def _entry() -> PluginCatalogEntry: "description": "Transform text records", "package": { "name": "data-designer-text-transform", - "version": "0.1.0", - "path": "plugins/data-designer-text-transform", + }, + "install": { + "requirement": "data-designer-text-transform", + "index_url": "https://docs.example.test/simple/", }, "entry_point": { "group": "data_designer.plugins", @@ -256,10 +265,6 @@ def _entry() -> PluginCatalogEntry: "marker": None, }, }, - "source": { - "type": "pypi", - "package": "data-designer-text-transform", - }, "docs": { "url": "https://docs.example.test/plugins/data-designer-text-transform/", }, diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py new file mode 100644 index 000000000..687492159 --- /dev/null +++ b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from data_designer.cli.plugin_catalog import ( + DEFAULT_PLUGIN_CATALOG_ALIAS, + DEFAULT_PLUGIN_CATALOG_URL_ENV_VAR, + PluginCatalogError, +) +from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository, normalize_catalog_location + + +def test_repository_includes_default_nvidia_catalog(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + + catalogs = repository.list_catalogs() + + assert [catalog.alias for catalog in catalogs] == [DEFAULT_PLUGIN_CATALOG_ALIAS] + assert catalogs[0].trusted is True + + +def test_default_catalog_honors_url_environment_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(DEFAULT_PLUGIN_CATALOG_URL_ENV_VAR, "https://example.test/catalog/plugins.json") + repository = PluginCatalogRepository(tmp_path) + + catalog = repository.default_catalog() + + assert catalog.url == "https://example.test/catalog/plugins.json" + + +def test_add_catalog_normalizes_github_repository_url(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + + catalog = repository.add_catalog("research", "https://github.com/acme/dd-plugins") + + assert catalog.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json" + assert repository.get_catalog("research") == catalog + + +def test_add_catalog_normalizes_github_tree_url_with_subdirectory(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + + catalog = repository.add_catalog("research", "https://github.com/acme/dd-plugins/tree/main/custom-catalog") + + assert catalog.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/custom-catalog/catalog/plugins.json" + + +def test_catalog_aliases_are_case_insensitive(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + + catalog = repository.add_catalog("Research", "https://github.com/acme/dd-plugins") + + assert repository.get_catalog("research") == catalog + with pytest.raises(ValueError, match="already exists"): + repository.add_catalog("research", "https://github.com/acme/other-plugins") + with pytest.raises(ValueError, match="already exists"): + repository.add_catalog("NVIDIA", "https://github.com/acme/nvidia-plugins") + + repository.remove_catalog("research") + + assert repository.get_catalog("Research") is None + + +def test_normalize_local_catalog_directory() -> None: + normalized = normalize_catalog_location("~/plugins") + + assert normalized.endswith("/plugins/catalog/plugins.json") + + +def test_load_catalog_uses_cache_when_source_is_unavailable(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + first_catalog = repository.load_catalog("local") + catalog_path.unlink() + cached_catalog = repository.load_catalog("local") + + assert first_catalog.plugins[0].name == "text-transform" + assert cached_catalog.plugins[0].name == "text-transform" + + +def test_load_catalog_with_zero_cache_ttl_refreshes_source(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path, plugin_name="text-transform") + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path), cache_ttl_seconds=0) + + first_catalog = repository.load_catalog("local") + catalog_path.write_text(json.dumps(_catalog_payload(plugin_name="fresh-transform"))) + refreshed_catalog = repository.load_catalog("local") + + assert first_catalog.plugins[0].name == "text-transform" + assert refreshed_catalog.plugins[0].name == "fresh-transform" + + +def test_load_catalog_cache_file_is_keyed_by_alias_and_url(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + repository.load_catalog("local") + + cache_files = list(repository.cache_dir.glob("*.json")) + assert len(cache_files) == 1 + assert cache_files[0].name.startswith("local-") + assert cache_files[0].name != "local.json" + + +def test_load_catalog_rejects_unsupported_schema_version(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path, schema_version=999) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="unsupported catalog schema_version"): + repository.load_catalog("local", refresh=True) + + +def test_load_catalog_accepts_schema_v2_package_catalog(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + packages=[ + _package_entry( + package_name="data-designer-index-package", + plugins=[ + _runtime_plugin("index-column", plugin_type="column-generator"), + _runtime_plugin("index-processor", plugin_type="processor"), + ], + install={ + "requirement": "data-designer-index-package", + "index_url": "https://docs.example.test/simple/", + }, + ), + _package_entry( + package_name="data-designer-git-plugin", + plugins=[_runtime_plugin("git-plugin", plugin_type="seed-reader")], + install={ + "requirement": ( + "data-designer-git-plugin @ " + "git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git@" + "data-designer-git-plugin/v0.1.0" + ), + }, + ), + _package_entry( + package_name="data-designer-url-plugin", + plugins=[_runtime_plugin("url-plugin", plugin_type="processor")], + install={ + "requirement": ( + "data-designer-url-plugin @ " + "https://packages.example.test/data_designer_url_plugin-0.1.0-py3-none-any.whl" + ), + }, + ), + ], + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + catalog = repository.load_catalog("local", refresh=True) + + assert [package.name for package in catalog.packages] == [ + "data-designer-index-package", + "data-designer-git-plugin", + "data-designer-url-plugin", + ] + assert [entry.name for entry in catalog.plugins] == [ + "index-column", + "index-processor", + "git-plugin", + "url-plugin", + ] + assert catalog.plugins[0].install.index_url == "https://docs.example.test/simple/" + + +def test_load_catalog_rejects_invalid_schema_v2_install_metadata(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + packages=[ + _package_entry( + package_name="data-designer-invalid-install", + plugins=[_runtime_plugin("invalid-install")], + install={ + "requirement": "data-designer-other", + "index_url": "https://docs.example.test/simple/", + }, + ) + ], + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="expected a requirement for 'data-designer-invalid-install'"): + repository.load_catalog("local", refresh=True) + + +def test_load_catalog_rejects_unexpected_schema_v2_fields(tmp_path: Path) -> None: + package = _package_entry() + package["tags"] = ["extra"] + catalog_path = _write_catalog(tmp_path, packages=[package]) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="catalog packages\\[0\\] has invalid fields"): + repository.load_catalog("local", refresh=True) + + +def test_load_catalog_rejects_duplicate_runtime_plugin_names(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + packages=[ + _package_entry( + package_name="data-designer-one", + plugins=[_runtime_plugin("duplicate", entry_point_name="first-entry")], + ), + _package_entry( + package_name="data-designer-two", + plugins=[_runtime_plugin("duplicate", entry_point_name="second-entry")], + ), + ], + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="duplicate runtime plugin name"): + repository.load_catalog("local", refresh=True) + + +def _write_catalog( + tmp_path: Path, + *, + schema_version: int = 2, + plugin_name: str = "text-transform", + packages: list[dict] | None = None, +) -> Path: + catalog_dir = tmp_path / "catalog" + catalog_dir.mkdir() + catalog_path = catalog_dir / "plugins.json" + catalog_path.write_text( + json.dumps(_catalog_payload(schema_version=schema_version, plugin_name=plugin_name, packages=packages)) + ) + return catalog_path + + +def _catalog_payload( + *, + schema_version: int = 2, + plugin_name: str = "text-transform", + packages: list[dict] | None = None, +) -> dict: + return { + "schema_version": schema_version, + "packages": packages if packages is not None else [_package_entry(plugins=[_runtime_plugin(plugin_name)])], + } + + +def _package_entry( + *, + package_name: str = "data-designer-text-transform", + plugins: list[dict] | None = None, + install: dict | None = None, +) -> dict: + return { + "name": package_name, + "description": f"{package_name} package", + "install": install + or { + "requirement": package_name, + "index_url": "https://docs.example.test/simple/", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": None, + }, + }, + "docs": { + "url": f"https://docs.example.test/plugins/{package_name}/", + }, + "plugins": plugins if plugins is not None else [_runtime_plugin("text-transform")], + } + + +def _runtime_plugin( + plugin_name: str, + *, + plugin_type: str = "processor", + entry_point_name: str | None = None, +) -> dict: + runtime_entry_point_name = plugin_name if entry_point_name is None else entry_point_name + return { + "name": plugin_name, + "plugin_type": plugin_type, + "entry_point": { + "group": "data_designer.plugins", + "name": runtime_entry_point_name, + "value": f"data_designer_{plugin_name.replace('-', '_')}.plugin:plugin", + }, + } diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py deleted file mode 100644 index dc26c9d69..000000000 --- a/packages/data-designer/tests/cli/repositories/test_plugin_tap_repository.py +++ /dev/null @@ -1,275 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from data_designer.cli.plugin_catalog import ( - DEFAULT_PLUGIN_TAP_ALIAS, - DEFAULT_PLUGIN_TAP_URL_ENV_VAR, - PluginCatalogError, -) -from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository, normalize_tap_location - - -def test_repository_includes_default_nvidia_tap(tmp_path: Path) -> None: - repository = PluginTapRepository(tmp_path) - - taps = repository.list_taps() - - assert [tap.alias for tap in taps] == [DEFAULT_PLUGIN_TAP_ALIAS] - assert taps[0].trusted is True - - -def test_default_tap_honors_url_environment_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(DEFAULT_PLUGIN_TAP_URL_ENV_VAR, "https://example.test/catalog/plugins.json") - repository = PluginTapRepository(tmp_path) - - tap = repository.default_tap() - - assert tap.url == "https://example.test/catalog/plugins.json" - - -def test_add_tap_normalizes_github_repository_url(tmp_path: Path) -> None: - repository = PluginTapRepository(tmp_path) - - tap = repository.add_tap("research", "https://github.com/acme/dd-plugins") - - assert tap.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json" - assert repository.get_tap("research") == tap - - -def test_add_tap_normalizes_github_tree_url_with_subdirectory(tmp_path: Path) -> None: - repository = PluginTapRepository(tmp_path) - - tap = repository.add_tap("research", "https://github.com/acme/dd-plugins/tree/main/custom-catalog") - - assert tap.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/custom-catalog/catalog/plugins.json" - - -def test_tap_aliases_are_case_insensitive(tmp_path: Path) -> None: - repository = PluginTapRepository(tmp_path) - - tap = repository.add_tap("Research", "https://github.com/acme/dd-plugins") - - assert repository.get_tap("research") == tap - with pytest.raises(ValueError, match="already exists"): - repository.add_tap("research", "https://github.com/acme/other-plugins") - with pytest.raises(ValueError, match="already exists"): - repository.add_tap("NVIDIA", "https://github.com/acme/nvidia-plugins") - - repository.remove_tap("research") - - assert repository.get_tap("Research") is None - - -def test_normalize_local_tap_directory() -> None: - normalized = normalize_tap_location("~/plugins") - - assert normalized.endswith("/plugins/catalog/plugins.json") - - -def test_load_catalog_uses_cache_when_source_is_unavailable(tmp_path: Path) -> None: - catalog_path = _write_catalog(tmp_path) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - first_catalog = repository.load_catalog("local") - catalog_path.unlink() - cached_catalog = repository.load_catalog("local") - - assert first_catalog.plugins[0].name == "text-transform" - assert cached_catalog.plugins[0].name == "text-transform" - - -def test_load_catalog_with_zero_cache_ttl_refreshes_source(tmp_path: Path) -> None: - catalog_path = _write_catalog(tmp_path, plugin_name="text-transform") - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path), cache_ttl_seconds=0) - - first_catalog = repository.load_catalog("local") - catalog_path.write_text(json.dumps(_catalog_payload(plugin_name="fresh-transform"))) - refreshed_catalog = repository.load_catalog("local") - - assert first_catalog.plugins[0].name == "text-transform" - assert refreshed_catalog.plugins[0].name == "fresh-transform" - - -def test_load_catalog_cache_file_is_keyed_by_alias_and_url(tmp_path: Path) -> None: - catalog_path = _write_catalog(tmp_path) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - repository.load_catalog("local") - - cache_files = list(repository.cache_dir.glob("*.json")) - assert len(cache_files) == 1 - assert cache_files[0].name.startswith("local-") - assert cache_files[0].name != "local.json" - - -def test_load_catalog_rejects_unsupported_schema_version(tmp_path: Path) -> None: - catalog_path = _write_catalog(tmp_path, schema_version=999) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - with pytest.raises(PluginCatalogError, match="unsupported catalog schema_version"): - repository.load_catalog("local", refresh=True) - - -def test_load_catalog_accepts_schema_v2_source_union(tmp_path: Path) -> None: - catalog_path = _write_catalog( - tmp_path, - plugins=[ - _plugin_entry( - "pypi-plugin", - package_name="data-designer-pypi-plugin", - source={"type": "pypi", "package": "data-designer-pypi-plugin"}, - ), - _plugin_entry( - "git-plugin", - package_name="data-designer-git-plugin", - source={ - "type": "git", - "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", - "ref": "data-designer-git-plugin/v0.1.0", - "subdirectory": "plugins/data-designer-git-plugin", - }, - ), - _plugin_entry( - "path-plugin", - package_name="data-designer-path-plugin", - source={ - "type": "path", - "path": "plugins/data-designer-path-plugin", - "editable": True, - }, - ), - ], - ) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - catalog = repository.load_catalog("local", refresh=True) - - assert [entry.name for entry in catalog.plugins] == ["pypi-plugin", "git-plugin", "path-plugin"] - - -def test_load_catalog_rejects_invalid_schema_v2_source(tmp_path: Path) -> None: - catalog_path = _write_catalog( - tmp_path, - plugins=[ - _plugin_entry( - "invalid-git-source", - package_name="data-designer-invalid-git-source", - source={ - "type": "git", - "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", - }, - ) - ], - ) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - with pytest.raises(PluginCatalogError, match="invalid 'git' source fields"): - repository.load_catalog("local", refresh=True) - - -def test_load_catalog_rejects_unexpected_schema_v2_fields(tmp_path: Path) -> None: - plugin = _plugin_entry("text-transform") - plugin["tags"] = ["extra"] - catalog_path = _write_catalog(tmp_path, plugins=[plugin]) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - with pytest.raises(PluginCatalogError, match="catalog plugins\\[0\\] has invalid fields"): - repository.load_catalog("local", refresh=True) - - -def test_load_catalog_rejects_duplicate_runtime_plugin_names(tmp_path: Path) -> None: - catalog_path = _write_catalog( - tmp_path, - plugins=[ - _plugin_entry("catalog-one", package_name="data-designer-one", entry_point_name="duplicate"), - _plugin_entry("catalog-two", package_name="data-designer-two", entry_point_name="duplicate"), - ], - ) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) - - with pytest.raises(PluginCatalogError, match="duplicate runtime plugin name"): - repository.load_catalog("local", refresh=True) - - -def _write_catalog( - tmp_path: Path, - *, - schema_version: int = 2, - plugin_name: str = "text-transform", - plugins: list[dict] | None = None, -) -> Path: - catalog_dir = tmp_path / "catalog" - catalog_dir.mkdir() - catalog_path = catalog_dir / "plugins.json" - catalog_path.write_text( - json.dumps(_catalog_payload(schema_version=schema_version, plugin_name=plugin_name, plugins=plugins)) - ) - return catalog_path - - -def _catalog_payload( - *, - schema_version: int = 2, - plugin_name: str = "text-transform", - plugins: list[dict] | None = None, -) -> dict: - return { - "schema_version": schema_version, - "plugins": plugins if plugins is not None else [_plugin_entry(plugin_name)], - } - - -def _plugin_entry( - plugin_name: str, - *, - package_name: str = "data-designer-text-transform", - entry_point_name: str | None = None, - source: dict | None = None, -) -> dict: - runtime_plugin_name = plugin_name if entry_point_name is None else entry_point_name - return { - "name": plugin_name, - "plugin_type": "processor", - "description": "Transform text records", - "package": { - "name": package_name, - "version": "0.1.0", - "path": f"plugins/{package_name}", - }, - "entry_point": { - "group": "data_designer.plugins", - "name": runtime_plugin_name, - "value": f"{package_name.replace('-', '_')}.plugin:plugin", - }, - "compatibility": { - "python": {"specifier": ">=3.10"}, - "data_designer": { - "requirement": "data-designer>=0.5.7", - "specifier": ">=0.5.7", - "marker": None, - }, - }, - "source": source - or { - "type": "pypi", - "package": package_name, - }, - "docs": { - "url": f"https://docs.example.test/plugins/{package_name}/", - }, - } diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index 98c876926..89e581661 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -11,7 +11,7 @@ import pytest from data_designer.cli.plugin_catalog import PluginCatalog, PluginCatalogEntry -from data_designer.cli.repositories.plugin_tap_repository import PluginTapRepository +from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository from data_designer.cli.services.plugin_catalog_service import PluginCatalogService @@ -78,38 +78,33 @@ def test_get_entry_rejects_incompatible_plugin_when_requested(tmp_path: Path) -> repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - with pytest.raises(ValueError, match="no compatible version"): + with pytest.raises(ValueError, match="not compatible"): service.get_entry("future-plugin", "local", include_incompatible=False) -def test_get_entry_prefers_newest_compatible_match_when_include_incompatible() -> None: - repository = Mock(spec=PluginTapRepository) +def test_get_entry_resolves_package_name() -> None: + repository = Mock(spec=PluginCatalogRepository) repository.load_catalog.return_value = PluginCatalog.model_validate( { "schema_version": 2, - "plugins": [ - _entry( - name="versioned-plugin", - plugin_type="processor", - package_name="data-designer-versioned-plugin", + "packages": [ + _package( + package_name="data-designer-package-target", data_designer_specifier=">=0.5.7", - package_version="0.2.0", - ), - _entry( - name="versioned-plugin", - plugin_type="processor", - package_name="data-designer-versioned-plugin", - data_designer_specifier=">=99.0", - package_version="99.0.0", + plugins=[ + _runtime_plugin(name="package-column", plugin_type="column-generator"), + _runtime_plugin(name="package-processor", plugin_type="processor"), + ], ), ], } ) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - entry = service.get_entry("versioned-plugin", "local", include_incompatible=True) + entry = service.get_entry("data-designer-package-target", "local", include_incompatible=True) - assert entry.package.version == "0.2.0" + assert entry.name == "package-column" + assert entry.package.name == "data-designer-package-target" def test_group_entries_by_package_groups_multi_plugin_packages(tmp_path: Path) -> None: @@ -168,7 +163,7 @@ def test_list_installed_plugins_uses_entry_point_metadata_without_loading_plugin group="data_designer.plugins", ) ] - service = PluginCatalogService(PluginTapRepository(tmp_path)) + service = PluginCatalogService(PluginCatalogRepository(tmp_path)) installed = service.list_installed_plugins() @@ -178,53 +173,86 @@ def test_list_installed_plugins_uses_entry_point_metadata_without_loading_plugin mock_entry_points.assert_called_once_with(group="data_designer.plugins") -def _repository_with_catalog(tmp_path: Path) -> PluginTapRepository: +def _repository_with_catalog(tmp_path: Path) -> PluginCatalogRepository: catalog_path = tmp_path / "plugins.json" catalog_path.write_text(json.dumps(_catalog_payload())) - repository = PluginTapRepository(tmp_path) - repository.add_tap("local", str(catalog_path)) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) return repository def _catalog_payload() -> dict: return { "schema_version": 2, - "plugins": [ - _entry( - name="compatible-plugin", - plugin_type="seed-reader", + "packages": [ + _package( package_name="data-designer-compatible-plugin", data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="compatible-plugin", plugin_type="seed-reader")], ), - _entry( - name="future-plugin", - plugin_type="processor", + _package( package_name="data-designer-future-plugin", data_designer_specifier=">=99.0", + plugins=[_runtime_plugin(name="future-plugin", plugin_type="processor")], ), - _entry( - name="shared-column", - plugin_type="column-generator", - package_name="data-designer-shared-package", - data_designer_specifier=">=0.5.7", - ), - _entry( - name="shared-processor", - plugin_type="processor", + _package( package_name="data-designer-shared-package", data_designer_specifier=">=0.5.7", + plugins=[ + _runtime_plugin(name="shared-column", plugin_type="column-generator"), + _runtime_plugin(name="shared-processor", plugin_type="processor"), + ], ), ], } +def _package( + *, + package_name: str, + data_designer_specifier: str, + plugins: list[dict], +) -> dict: + return { + "name": package_name, + "description": f"{package_name} description", + "install": { + "requirement": package_name, + "index_url": "https://docs.example.test/simple/", + }, + "compatibility": { + "python": {"specifier": ">=3.10"}, + "data_designer": { + "requirement": f"data-designer{data_designer_specifier}", + "specifier": data_designer_specifier, + "marker": None, + }, + }, + "docs": { + "url": f"https://docs.example.test/plugins/{package_name}/", + }, + "plugins": plugins, + } + + +def _runtime_plugin(*, name: str, plugin_type: str) -> dict: + return { + "name": name, + "plugin_type": plugin_type, + "entry_point": { + "group": "data_designer.plugins", + "name": name, + "value": f"data_designer_{name.replace('-', '_')}.plugin:plugin", + }, + } + + def _entry( *, name: str, plugin_type: str, package_name: str, data_designer_specifier: str, - package_version: str = "0.1.0", ) -> dict: return { "name": name, @@ -232,8 +260,10 @@ def _entry( "description": f"{name} description", "package": { "name": package_name, - "version": package_version, - "path": f"plugins/{package_name}", + }, + "install": { + "requirement": package_name, + "index_url": "https://docs.example.test/simple/", }, "entry_point": { "group": "data_designer.plugins", @@ -248,10 +278,6 @@ def _entry( "marker": None, }, }, - "source": { - "type": "pypi", - "package": package_name, - }, "docs": { "url": f"https://docs.example.test/plugins/{package_name}/", }, diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index fdd120dee..d7eb8e984 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -4,116 +4,94 @@ from __future__ import annotations import sys -from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock, patch import pytest -from data_designer.cli.plugin_catalog import PluginCatalogEntry, PluginTapConfig +from data_designer.cli.plugin_catalog import PluginCatalogConfig, PluginCatalogEntry from data_designer.cli.services.plugin_install_service import PluginInstallService -def test_build_pypi_install_plan_uses_exact_catalog_version() -> None: - entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) - tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") +def test_build_pip_install_plan_uses_requirement_and_extra_index() -> None: + entry = _entry( + package_name="data-designer-template", + install={ + "requirement": "data-designer-template", + "index_url": "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + }, + ) + catalog = PluginCatalogConfig( + alias="nvidia", url="https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json" + ) service = PluginInstallService() - plan = service.build_install_plan(entry, tap, manager="pip") + plan = service.build_install_plan(entry, catalog, manager="pip") assert plan.command == [ sys.executable, "-m", "pip", "install", - "data-designer-text-transform==0.1.0", + "--extra-index-url", + "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + "data-designer-template", ] - assert plan.source_description == "data-designer-text-transform==0.1.0" - - -def test_build_git_install_plan_includes_ref_and_subdirectory() -> None: - entry = _entry( - source={ - "type": "git", - "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", - "ref": "data-designer-text-transform/v0.1.0", - "subdirectory": "plugins/data-designer-text-transform", - } + assert plan.source_description == ( + "data-designer-template via https://nvidia-nemo.github.io/DataDesignerPlugins/simple/" ) - tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") - service = PluginInstallService() - plan = service.build_install_plan(entry, tap, manager="pip") - assert plan.command[-1] == ( - "data-designer-text-transform @ git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git" - "@data-designer-text-transform/v0.1.0" - "#subdirectory=plugins/data-designer-text-transform" +def test_build_direct_reference_install_plan_uses_requirement_verbatim() -> None: + requirement = ( + "data-designer-template @ " + "git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git@data-designer-template/v0.1.0" ) - - -@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") -def test_build_uv_install_plan_targets_current_python(mock_which: Mock) -> None: - entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) - tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") + entry = _entry(package_name="data-designer-template", install={"requirement": requirement}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") service = PluginInstallService() - plan = service.build_install_plan(entry, tap, manager="uv") + plan = service.build_install_plan(entry, catalog, manager="pip") - assert plan.command[:5] == ["uv", "pip", "install", "--python", sys.executable] + assert plan.command[-1] == requirement + assert "--extra-index-url" not in plan.command -def test_build_git_install_plan_requires_ref_and_subdirectory() -> None: +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_uv_install_plan_targets_current_python_and_adds_catalog_index(mock_which: Mock) -> None: entry = _entry( - source={ - "type": "git", - "url": "https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git", - } + package_name="data-designer-template", + install={ + "requirement": "data-designer-template", + "index_url": "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + }, ) - tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") - service = PluginInstallService() - - with pytest.raises(ValueError, match="requires 'ref'"): - service.build_install_plan(entry, tap, manager="pip") - - -def test_build_path_install_plan_resolves_relative_path_from_local_tap_root(tmp_path: Path) -> None: - entry = _entry( - source={ - "type": "path", - "path": "plugins/data-designer-text-transform", - "editable": True, - } + catalog = PluginCatalogConfig( + alias="nvidia", url="https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json" ) - tap = PluginTapConfig(alias="local", url=str(tmp_path / "catalog" / "plugins.json")) service = PluginInstallService() - plan = service.build_install_plan(entry, tap, manager="pip") + plan = service.build_install_plan(entry, catalog, manager="uv") assert plan.command == [ - sys.executable, - "-m", + "uv", "pip", "install", - "-e", - str(tmp_path / "plugins" / "data-designer-text-transform"), + "--python", + sys.executable, + "--default-index", + "https://pypi.org/simple/", + "--index", + "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + "data-designer-template", ] -def test_build_install_plan_requires_source() -> None: - entry = _entry(source=None) - tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") - service = PluginInstallService() - - with pytest.raises(ValueError, match="does not declare a source"): - service.build_install_plan(entry, tap, manager="pip") - - def test_install_raises_when_runner_fails() -> None: service = PluginInstallService(runner=lambda command: 2) - entry = _entry(source={"type": "pypi", "package": "data-designer-text-transform"}) - tap = PluginTapConfig(alias="local", url="/catalog/plugins.json") - plan = service.build_install_plan(entry, tap, manager="pip") + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + plan = service.build_install_plan(entry, catalog, manager="pip") with pytest.raises(RuntimeError, match="status 2"): service.install(plan) @@ -126,9 +104,10 @@ def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( mock_entry_points: Mock, ) -> None: entry = _entry( - source={"type": "pypi", "package": "data-designer-text-transform"}, + package_name="data-designer-template", plugin_name="text-transform-v2", entry_point_name="text-transform", + install={"requirement": "data-designer-template"}, ) mock_entry_points.return_value = [ SimpleNamespace(name="other-plugin"), @@ -141,9 +120,32 @@ def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( mock_entry_points.assert_called_once_with(group="data_designer.plugins") +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") +def test_verify_entry_points_requires_every_declared_entry_point(mock_entry_points: Mock) -> None: + entries = [ + _entry( + package_name="data-designer-retrieval-sdg", + plugin_name="document-chunker", + entry_point_name="document-chunker", + install={"requirement": "data-designer-retrieval-sdg"}, + ), + _entry( + package_name="data-designer-retrieval-sdg", + plugin_name="embedding-dedup", + entry_point_name="embedding-dedup", + install={"requirement": "data-designer-retrieval-sdg"}, + ), + ] + mock_entry_points.return_value = [SimpleNamespace(name="document-chunker")] + service = PluginInstallService() + + assert service.verify_entry_points(entries) is False + + def _entry( - source: dict | None, *, + package_name: str, + install: dict, plugin_name: str = "text-transform", entry_point_name: str = "text-transform", ) -> PluginCatalogEntry: @@ -152,14 +154,13 @@ def _entry( "plugin_type": "processor", "description": "Transform text records", "package": { - "name": "data-designer-text-transform", - "version": "0.1.0", - "path": "plugins/data-designer-text-transform", + "name": package_name, }, + "install": install, "entry_point": { "group": "data_designer.plugins", "name": entry_point_name, - "value": "data_designer_text_transform.plugin:plugin", + "value": "data_designer_template.plugin:plugin", }, "compatibility": { "python": {"specifier": ">=3.10"}, @@ -169,9 +170,8 @@ def _entry( "marker": None, }, }, - "source": source, "docs": { - "url": "https://docs.example.test/plugins/data-designer-text-transform/", + "url": f"https://docs.example.test/plugins/{package_name}/", }, } return PluginCatalogEntry.model_validate(payload) From d3c7320880542699b4adcf1178064d813c4f12fe Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 18:30:53 +0000 Subject: [PATCH 12/34] tidy plugin catalog workflow docs --- packages/data-designer/src/data_designer/cli/README.md | 2 +- .../data_designer/cli/controllers/plugin_catalog_controller.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 66d69c442..ceebb9b0f 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -67,7 +67,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - **Files**: - `model_controller.py`: Orchestrates model configuration workflows - `provider_controller.py`: Orchestrates provider configuration workflows - - `plugin_catalog_controller.py`: Orchestrates plugin catalog, catalog, and install workflows + - `plugin_catalog_controller.py`: Orchestrates plugin catalog browsing, alias management, and install workflows **Key Features**: - **Associated Resource Management**: When deleting a provider, the controller checks for associated models and prompts the user to delete them together diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index eb26ebb49..dde0d14a0 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -35,7 +35,7 @@ class PluginCatalogController: - """Controller for plugin catalog, catalog, and install workflows. + """Controller for plugin catalog browsing, alias management, and install workflows. Catalog browsing and environment mutation intentionally use separate services so read-only catalog operations stay decoupled from package-manager execution. From 429d549c3f051e53317ea03d4d678b6bfe21196d Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 18:59:46 +0000 Subject: [PATCH 13/34] align plugin catalog CLI with package contract --- architecture/cli.md | 38 ++- .../src/data_designer/cli/README.md | 6 +- .../src/data_designer/cli/commands/plugins.py | 30 ++- .../controllers/plugin_catalog_controller.py | 198 +++++++++++----- .../src/data_designer/cli/main.py | 6 +- .../src/data_designer/cli/plugin_catalog.py | 39 ++-- .../repositories/plugin_catalog_repository.py | 41 +++- .../cli/services/plugin_install_service.py | 49 +++- .../cli/commands/test_plugins_command.py | 20 +- .../test_plugin_catalog_controller.py | 216 +++++++++++++++++- .../test_plugin_catalog_repository.py | 164 +++++++++++++ .../services/test_plugin_catalog_service.py | 86 ++++++- .../services/test_plugin_install_service.py | 150 +++++++++++- packages/data-designer/tests/cli/test_main.py | 55 +++++ 14 files changed, 976 insertions(+), 122 deletions(-) diff --git a/architecture/cli.md b/architecture/cli.md index eba2a3ca7..3660c9e37 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -1,12 +1,12 @@ # CLI -The CLI (`data-designer`) provides an interactive command-line interface for configuring models, providers, tools, and personas, as well as running dataset generation. It uses a layered architecture for config management and delegates generation to the public `DataDesigner` API. +The CLI (`data-designer`) provides an interactive command-line interface for configuring models, providers, tools, and personas, discovering/installing plugins from catalogs, and running dataset generation. It uses a layered architecture for setup workflows and delegates generation to the public `DataDesigner` API. Source: `packages/data-designer/src/data_designer/cli/` ## Overview -The CLI is built on Typer with lazy command loading to keep startup fast. Config management commands follow a **command → controller → service → repository** layering pattern. Generation commands bypass this stack and use the public `DataDesigner` class directly. +The CLI is built on Typer with lazy command loading to keep startup fast. Config management and plugin catalog commands follow a **command → controller → service → repository** layering pattern. Generation commands bypass this stack and use the public `DataDesigner` class directly. ## Key Components @@ -20,7 +20,7 @@ The CLI is built on Typer with lazy command loading to keep startup fast. Config `create_lazy_typer_group` and `_LazyCommand` stubs defer importing command modules until a command is actually invoked. This keeps `data-designer --help` fast — only the command names and descriptions are loaded eagerly; the full module (and its dependencies) loads on first use. -### Layering Pattern (Config Management) +### Layering Pattern (Setup Workflows) Config management commands (models, providers, tools, personas) follow a consistent four-layer pattern: @@ -35,6 +35,17 @@ Repositories: `ModelRepository`, `ProviderRepository`, `ToolRepository`, `MCPPro Services mirror the repository domains with business logic (validation, conflict resolution). +Plugin catalog commands use the same layering shape: + +| Layer | Role | Example | +|-------|------|---------| +| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugins list/search/info/install/installed/catalogs` → `PluginCatalogController(DATA_DESIGNER_HOME)` | +| **Controller** | UX flow: catalog tables, package metadata, compatibility display, install confirmation | `PluginCatalogController` composes catalog + install services | +| **Service** | Domain rules: package-first flattening, compatibility checks, install planning, entry point verification | `PluginCatalogService`, `PluginInstallService` | +| **Repository** | File/cache I/O for catalog aliases and catalog documents | `PluginCatalogRepository` | + +The built-in `nvidia` catalog points at `https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`. `NVIDIA-NeMo/DataDesignerPlugins` defines the package-first catalog shape: top-level packages carry install metadata, compatibility constraints, docs, and nested runtime plugins. The CLI flattens nested plugins for list/search display, but `info` and `install` resolve back to the package so installation targets the package requirement. + ### Generation Commands `preview`, `create`, and `validate` commands use `GenerationController`, which: @@ -62,6 +73,24 @@ User invokes command (e.g., `data-designer config models`) → Repository reads/writes config files ``` +### Plugin Catalog Discovery +``` +User invokes command (e.g., `data-designer plugins list`) + → Command function wires DATA_DESIGNER_HOME and catalog options + → PluginCatalogController resolves the catalog alias + → PluginCatalogService loads and filters package-first catalog entries + → PluginCatalogRepository reads local config and cached/remote catalog JSON +``` + +### Plugin Install +``` +User invokes command (e.g., `data-designer plugins install text-transform`) + → PluginCatalogController resolves runtime plugin or package name + → PluginCatalogService evaluates Python and Data Designer compatibility + → PluginInstallService builds a pip/uv install plan for the package requirement + → PluginInstallService verifies declared entry points after installation +``` + ### Generation ``` User invokes command (e.g., `data-designer create config.yaml`) @@ -73,8 +102,9 @@ User invokes command (e.g., `data-designer create config.yaml`) ## Design Decisions - **Lazy command loading** keeps `data-designer --help` responsive: command modules (and their heavy dependencies, such as the engine and model stacks) load only when a command is invoked, not at process startup. -- **Controller/service/repo for config, direct API for generation** — config management benefits from the layered pattern (testable services, swappable repositories). Generation doesn't need this indirection; it delegates to the same `DataDesigner` class that Python users call directly. +- **Controller/service/repo for setup workflows, direct API for generation** — config and plugin catalog workflows benefit from the layered pattern (testable services, swappable repositories). Generation doesn't need this indirection; it delegates to the same `DataDesigner` class that Python users call directly. - **`DATA_DESIGNER_HOME`** centralizes all CLI-managed state (model configs, provider configs, tool configs, personas) in a single directory, defaulting to `~/.data_designer/`. +- **Package-first plugin catalogs** keep install metadata at the package boundary while allowing one package to expose multiple runtime plugins through entry points. - **Rich-based UI** provides formatted tables, progress bars, and interactive prompts without requiring a web interface. ## Cross-References diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index ceebb9b0f..179828a62 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -279,7 +279,7 @@ data-designer config reset ### Discover and Install Plugins ```bash -# List compatible plugins from the default NVIDIA catalog +# List compatible plugin packages from the default NVIDIA catalog data-designer plugins list # Search a specific catalog @@ -288,7 +288,7 @@ data-designer plugins --catalog research search transform # Show metadata, compatibility, docs, and exact install command data-designer plugins info text-transform -# Install a plugin package from a catalog and verify declared entry point discovery +# Install a plugin package from a catalog and verify declared runtime entry point discovery data-designer plugins install text-transform --yes # Preview the install command without mutating the environment @@ -299,6 +299,6 @@ data-designer plugins catalogs add research https://github.com/acme/dd-plugins data-designer plugins catalogs list data-designer plugins catalogs remove research -# List installed plugin entry points without importing plugin modules +# List installed runtime plugin entry points without importing plugin modules data-designer plugins installed ``` diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugins.py index 17ab6a4ab..fae7adb1f 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugins.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugins.py @@ -26,10 +26,10 @@ def list_command( include_incompatible: bool = typer.Option( False, "--include-incompatible", - help="Show plugins that do not satisfy the local Python or Data Designer version.", + help="Show catalog packages that do not satisfy the local Python or Data Designer version.", ), ) -> None: - """List discoverable Data Designer plugins from a catalog.""" + """List installable Data Designer plugin packages from a catalog.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_list( catalog_alias=_resolve_catalog_alias(ctx, catalog), @@ -41,7 +41,7 @@ def list_command( def search_command( ctx: typer.Context, query: str = typer.Argument( - help="Keyword, plugin type, package name, requirement, docs URL, or entry point to search for." + help="Keyword, runtime plugin name or type, package name, requirement, docs URL, or entry point to search for." ), catalog: str | None = typer.Option( None, @@ -56,10 +56,10 @@ def search_command( include_incompatible: bool = typer.Option( False, "--include-incompatible", - help="Search plugins that do not satisfy the local Python or Data Designer version.", + help="Search catalog packages that do not satisfy the local Python or Data Designer version.", ), ) -> None: - """Search discoverable Data Designer plugins from a catalog.""" + """Search installable Data Designer plugin packages from a catalog.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_search( query, @@ -71,7 +71,10 @@ def search_command( def info_command( ctx: typer.Context, - plugin_name: str = typer.Argument(help="Runtime plugin name or package name from the catalog."), + package_or_plugin: str = typer.Argument( + help="Package name or runtime plugin name from the catalog.", + metavar="PACKAGE_OR_PLUGIN", + ), catalog: str | None = typer.Option( None, "--catalog", @@ -86,7 +89,7 @@ def info_command( """Show metadata, compatibility, docs, and install plan for one plugin package.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_info( - plugin_name, + package_or_plugin, catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, ) @@ -94,7 +97,10 @@ def info_command( def install_command( ctx: typer.Context, - plugin_name: str = typer.Argument(help="Runtime plugin name or package name from the catalog."), + package_or_plugin: str = typer.Argument( + help="Package name or runtime plugin name from the catalog.", + metavar="PACKAGE_OR_PLUGIN", + ), catalog: str | None = typer.Option( None, "--catalog", @@ -125,13 +131,13 @@ def install_command( force: bool = typer.Option( False, "--force", - help="Allow installation when only incompatible catalog entries are available.", + help="Allow installing a catalog package when compatibility checks fail.", ), ) -> None: """Install one Data Designer plugin package, then verify runtime discovery.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_install( - plugin_name, + package_or_plugin, catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, manager=manager, @@ -142,8 +148,8 @@ def install_command( def installed_command(ctx: typer.Context) -> None: - """List installed Data Designer plugin entry points.""" - _warn_if_parent_catalog_unused(ctx, "installed plugins are discovered from the current Python environment") + """List installed Data Designer runtime plugin entry points.""" + _warn_if_parent_catalog_unused(ctx, "installed runtime plugins are discovered from the current Python environment") controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_installed() diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index dde0d14a0..943737fa4 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -17,6 +17,7 @@ InstalledPluginInfo, PluginCatalogConfig, PluginCatalogEntry, + PluginCatalogError, ) from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository from data_designer.cli.services.plugin_catalog_service import PluginCatalogService @@ -54,16 +55,16 @@ def run_list( refresh: bool = False, include_incompatible: bool = False, ) -> None: - """List plugins from a catalog.""" + """List plugin packages from a catalog.""" catalog = self._get_catalog_or_exit(catalog_alias) entries = self._list_entries_or_exit(catalog.alias, refresh=refresh, include_incompatible=include_incompatible) - print_header("Data Designer Plugins") + print_header("Data Designer Plugin Packages") print_info(f"Catalog: {catalog.alias} ({catalog.url})") console.print() if not entries: - print_warning("No plugins found") + self._display_empty_list_state(catalog.alias, include_incompatible=include_incompatible) return self._display_catalog_entries(entries) @@ -76,41 +77,41 @@ def run_search( refresh: bool = False, include_incompatible: bool = False, ) -> None: - """Search plugins from a catalog.""" + """Search plugin packages from a catalog.""" catalog = self._get_catalog_or_exit(catalog_alias) - try: - entries = self.catalog_service.search_entries( - query, - catalog.alias, - refresh=refresh, - include_incompatible=include_incompatible, - ) - except Exception as e: - print_error(f"Failed to search plugin catalog: {e}") - raise typer.Exit(code=1) + entries = self._search_entries_or_exit( + query, + catalog.alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) - print_header("Data Designer Plugin Search") + print_header("Data Designer Plugin Package Search") print_info(f"Catalog: {catalog.alias} ({catalog.url})") print_info(f"Query: {query}") console.print() if not entries: - print_warning("No matching plugins found") + self._display_empty_search_state( + query, + catalog.alias, + include_incompatible=include_incompatible, + ) return self._display_catalog_entries(entries) def run_info( self, - plugin_name: str, + package_or_plugin: str, *, catalog_alias: str | None = None, refresh: bool = False, ) -> None: - """Show full metadata for one plugin.""" + """Show full metadata for one plugin package.""" catalog = self._get_catalog_or_exit(catalog_alias) - entry = self._get_entry_or_exit(plugin_name, catalog.alias, refresh=refresh) - package_entries = self.catalog_service.get_package_entries( + entry = self._get_entry_or_exit(package_or_plugin, catalog.alias, refresh=refresh) + package_entries = self._get_package_entries_or_exit( entry.package.name, catalog.alias, refresh=refresh, @@ -135,7 +136,10 @@ def run_info( console.print() display_config_preview( { - "package": entry.package.name, + "package": { + "name": entry.package.name, + "description": entry.description, + }, "install": entry.install.model_dump(mode="json", exclude_none=True), "compatibility": ( entry.compatibility.model_dump(mode="json", exclude_none=True) @@ -143,14 +147,14 @@ def run_info( else None ), "docs": entry.docs.model_dump(mode="json", exclude_none=True) if entry.docs is not None else None, - "plugins": [plugin.model_dump(mode="json", exclude_none=True) for plugin in package_entries], + "plugins": [_runtime_plugin_metadata(plugin) for plugin in package_entries], }, "Plugin Metadata", ) def run_install( self, - plugin_name: str, + package_or_plugin: str, *, catalog_alias: str | None = None, refresh: bool = False, @@ -159,10 +163,10 @@ def run_install( dry_run: bool = False, force: bool = False, ) -> None: - """Install one plugin from a catalog entry.""" + """Install one plugin package from the catalog.""" catalog = self._get_catalog_or_exit(catalog_alias) - entry = self._get_entry_or_exit(plugin_name, catalog.alias, refresh=refresh, include_incompatible=True) - package_entries = self.catalog_service.get_package_entries( + entry = self._get_entry_or_exit(package_or_plugin, catalog.alias, refresh=refresh, include_incompatible=True) + package_entries = self._get_package_entries_or_exit( entry.package.name, catalog.alias, refresh=refresh, @@ -170,8 +174,8 @@ def run_install( ) or [entry] compatibility = self.catalog_service.evaluate_compatibility(entry) - if not compatibility.is_compatible and not force: - print_error(f"Plugin {entry.name!r} is not compatible with this environment") + if not compatibility.is_compatible and not force and not dry_run: + print_error(f"Plugin package {entry.package.name!r} is not compatible with this environment") for reason in compatibility.reasons: console.print(f" - {reason}") raise typer.Exit(code=1) @@ -182,7 +186,7 @@ def run_install( print_error(f"Failed to build plugin install plan: {e}") raise typer.Exit(code=1) - print_header("Install Data Designer Plugin") + print_header("Install Data Designer Plugin Package") console.print(f" Package: [bold]{entry.package.name}[/bold]") console.print(f" Runtime plugins: [bold]{_format_runtime_plugins(package_entries)}[/bold]") console.print(f" Catalog: [bold]{catalog.alias}[/bold] ({catalog.url})") @@ -198,10 +202,15 @@ def run_install( ) if dry_run: - print_info("Dry run complete; no changes made") + if not compatibility.is_compatible and not force: + print_warning( + "Dry run complete; no changes made. A real install would be blocked unless you pass --force." + ) + else: + print_info("Dry run complete; no changes made") return - if not yes and not confirm_action("Install this plugin into the current Python environment?", default=False): + if not yes and not confirm_action("Install this package into the current Python environment?", default=False): print_info("No changes made") return @@ -221,18 +230,23 @@ def run_install( ) def run_installed(self) -> None: - """List installed plugin entry points without importing plugin modules.""" - print_header("Installed Data Designer Plugins") + """List installed runtime plugin entry points without importing plugin modules.""" + print_header("Installed Data Designer Runtime Plugins") installed_plugins = self.catalog_service.list_installed_plugins() if not installed_plugins: - print_warning("No installed Data Designer plugins were discovered") + print_warning("No installed Data Designer runtime plugins were discovered") return self._display_installed_plugins(installed_plugins) def run_catalogs_list(self) -> None: """List configured plugin catalogs.""" print_header("Data Designer Plugin Catalogs") - catalogs = self.catalog_service.list_catalogs() + try: + catalogs = self.catalog_service.list_catalogs() + except (PluginCatalogError, OSError) as e: + print_error(f"Failed to list plugin catalogs: {e}") + raise typer.Exit(code=1) + table = Table(title="Plugin Catalogs", border_style=NordColor.NORD8.value) table.add_column("Alias", style=NordColor.NORD14.value, no_wrap=True) table.add_column("URL", style=NordColor.NORD4.value) @@ -270,7 +284,7 @@ def run_catalogs_add( else: print_error(f"Invalid plugin catalog configuration: {e}") raise typer.Exit(code=1) - except Exception as e: + except (PluginCatalogError, OSError, ValueError) as e: print_error(f"Failed to add plugin catalog: {e}") raise typer.Exit(code=1) @@ -281,7 +295,7 @@ def run_catalogs_remove(self, *, alias: str) -> None: """Remove a plugin catalog alias.""" try: self.catalog_service.remove_catalog(alias) - except Exception as e: + except (PluginCatalogError, OSError, ValueError) as e: print_error(f"Failed to remove plugin catalog: {e}") raise typer.Exit(code=1) print_success(f"Plugin catalog {alias!r} removed") @@ -289,7 +303,7 @@ def run_catalogs_remove(self, *, alias: str) -> None: def _get_catalog_or_exit(self, catalog_alias: str | None) -> PluginCatalogConfig: try: return self.catalog_service.get_catalog(catalog_alias or DEFAULT_PLUGIN_CATALOG_ALIAS) - except ValueError as e: + except (PluginCatalogError, OSError, ValueError) as e: print_error(str(e)) raise typer.Exit(code=1) @@ -306,13 +320,32 @@ def _list_entries_or_exit( refresh=refresh, include_incompatible=include_incompatible, ) - except Exception as e: + except (PluginCatalogError, OSError, ValueError) as e: print_error(f"Failed to load plugin catalog: {e}") raise typer.Exit(code=1) + def _search_entries_or_exit( + self, + query: str, + catalog_alias: str, + *, + refresh: bool, + include_incompatible: bool, + ) -> list[PluginCatalogEntry]: + try: + return self.catalog_service.search_entries( + query, + catalog_alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + except (PluginCatalogError, OSError, ValueError) as e: + print_error(f"Failed to search plugin catalog: {e}") + raise typer.Exit(code=1) + def _get_entry_or_exit( self, - plugin_name: str, + package_or_plugin: str, catalog_alias: str, *, refresh: bool, @@ -320,30 +353,85 @@ def _get_entry_or_exit( ) -> PluginCatalogEntry: try: return self.catalog_service.get_entry( - plugin_name, + package_or_plugin, catalog_alias, refresh=refresh, include_incompatible=include_incompatible, ) - except Exception as e: + except (PluginCatalogError, OSError, ValueError) as e: print_error(str(e)) raise typer.Exit(code=1) + def _get_package_entries_or_exit( + self, + package_name: str, + catalog_alias: str, + *, + refresh: bool, + include_incompatible: bool, + ) -> list[PluginCatalogEntry]: + try: + return self.catalog_service.get_package_entries( + package_name, + catalog_alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + except (PluginCatalogError, OSError, ValueError) as e: + print_error(f"Failed to load plugin package metadata: {e}") + raise typer.Exit(code=1) + + def _display_empty_list_state(self, catalog_alias: str, *, include_incompatible: bool) -> None: + if include_incompatible: + print_warning("No plugin packages found") + return + + all_entries = self._list_entries_or_exit(catalog_alias, refresh=False, include_incompatible=True) + if all_entries: + print_warning("No compatible plugin packages found") + print_info("Incompatible catalog packages are hidden. Use --include-incompatible to show them.") + return + + print_warning("No plugin packages found") + + def _display_empty_search_state( + self, + query: str, + catalog_alias: str, + *, + include_incompatible: bool, + ) -> None: + if include_incompatible: + print_warning("No matching plugin packages found") + return + + all_matches = self._search_entries_or_exit( + query, + catalog_alias, + refresh=False, + include_incompatible=True, + ) + if all_matches: + print_warning("No compatible plugin packages matched") + print_info("Matching incompatible catalog packages are hidden. Use --include-incompatible to show them.") + return + + print_warning("No matching plugin packages found") + def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: - table = Table(title="Catalog Plugins", border_style=NordColor.NORD8.value) - table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) - table.add_column("Type", style=NordColor.NORD9.value, no_wrap=True) - table.add_column("Package", style=NordColor.NORD4.value, no_wrap=True) + table = Table(title="Catalog Plugin Packages", border_style=NordColor.NORD8.value) + table.add_column("Package", style=NordColor.NORD14.value, no_wrap=True) + table.add_column("Runtime Plugins", style=NordColor.NORD9.value) table.add_column("Compatible", style=NordColor.NORD13.value, no_wrap=True) table.add_column("Docs", style=NordColor.NORD7.value) - for entry in entries: + for package_entries in self.catalog_service.group_entries_by_package(entries).values(): + entry = package_entries[0] compatibility = self.catalog_service.evaluate_compatibility(entry) docs_url = entry.docs.url if entry.docs is not None and entry.docs.url is not None else "" table.add_row( - entry.name, - entry.plugin_type.value, entry.package.name, + _format_runtime_plugins(package_entries), "yes" if compatibility.is_compatible else "no", docs_url, ) @@ -351,8 +439,8 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: @staticmethod def _display_installed_plugins(installed_plugins: list[InstalledPluginInfo]) -> None: - table = Table(title="Installed Plugins", border_style=NordColor.NORD8.value) - table.add_column("Name", style=NordColor.NORD14.value, no_wrap=True) + table = Table(title="Installed Runtime Plugins", border_style=NordColor.NORD8.value) + table.add_column("Runtime Plugin", style=NordColor.NORD14.value, no_wrap=True) table.add_column("Entry Point", style=NordColor.NORD4.value) for plugin in installed_plugins: @@ -375,3 +463,11 @@ def _display_compatibility(compatibility: CompatibilityResult) -> None: def _format_runtime_plugins(entries: list[PluginCatalogEntry]) -> str: return ", ".join(f"{entry.name} ({entry.plugin_type.value})" for entry in entries) + + +def _runtime_plugin_metadata(entry: PluginCatalogEntry) -> dict[str, object]: + return { + "name": entry.name, + "plugin_type": entry.plugin_type.value, + "entry_point": entry.entry_point.model_dump(mode="json", exclude_none=True), + } diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index c1548aa3a..150f7f634 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -150,12 +150,12 @@ def _is_version_request(args: list[str]) -> bool: "list": { "module": f"{_CMD}.plugins", "attr": "list_command", - "help": "List plugins from a catalog", + "help": "List plugin packages from a catalog", }, "search": { "module": f"{_CMD}.plugins", "attr": "search_command", - "help": "Search plugins from a catalog", + "help": "Search plugin packages from a catalog", }, "info": { "module": f"{_CMD}.plugins", @@ -170,7 +170,7 @@ def _is_version_request(args: list[str]) -> bool: "installed": { "module": f"{_CMD}.plugins", "attr": "installed_command", - "help": "List installed plugin entry points", + "help": "List installed runtime plugin entry points", }, } ), diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index b9a864e71..36ee0cfed 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -53,7 +53,7 @@ class PluginCatalogError(ValueError): class PluginCompatibilityTarget(BaseModel): """Version requirement for one environment target.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") requirement: str | None = None specifier: str | None = None @@ -63,7 +63,7 @@ class PluginCompatibilityTarget(BaseModel): class PluginCompatibility(BaseModel): """Compatibility requirements declared by a catalog package.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") python: PluginCompatibilityTarget | None = None data_designer: PluginCompatibilityTarget | None = None @@ -72,7 +72,7 @@ class PluginCompatibility(BaseModel): class PluginPackageInfo(BaseModel): """Python distribution metadata for a catalog entry.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") name: str @@ -80,7 +80,7 @@ class PluginPackageInfo(BaseModel): class PluginEntryPointInfo(BaseModel): """Runtime entry point exposed by an installable plugin package.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") group: str = PLUGIN_ENTRY_POINT_GROUP name: str @@ -90,7 +90,7 @@ class PluginEntryPointInfo(BaseModel): class PluginInstallInfo(BaseModel): """Resolver-native install metadata for a catalog package.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") requirement: str index_url: str | None = None @@ -99,7 +99,7 @@ class PluginInstallInfo(BaseModel): class PluginDocsInfo(BaseModel): """Documentation metadata for a catalog package.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") url: str | None = None @@ -107,7 +107,7 @@ class PluginDocsInfo(BaseModel): class PluginCatalogEntry(BaseModel): """One discoverable runtime plugin entry from a catalog package.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") name: str plugin_type: PluginType @@ -122,7 +122,7 @@ class PluginCatalogEntry(BaseModel): class PluginCatalogRuntimePlugin(BaseModel): """Runtime plugin metadata nested under one catalog package.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") name: str plugin_type: PluginType @@ -132,7 +132,7 @@ class PluginCatalogRuntimePlugin(BaseModel): class PluginCatalogPackage(BaseModel): """One installable package from a package-first plugin catalog.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") name: str description: str = "" @@ -162,7 +162,7 @@ def entries(self) -> list[PluginCatalogEntry]: class PluginCatalog(BaseModel): """Versioned plugin catalog.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="forbid") schema_version: int packages: list[PluginCatalogPackage] = Field(default_factory=list) @@ -251,9 +251,21 @@ def _validate_plugin_catalog_payload(payload: object) -> None: if not isinstance(packages, list): raise PluginCatalogError("catalog document has invalid packages; expected a list") + package_names: dict[str, str] = {} runtime_names: dict[str, tuple[str, str]] = {} for index, raw_package in enumerate(packages): - for package_name, plugin_name, entry_point_name in _validate_catalog_package(raw_package, index): + validated_plugins = _validate_catalog_package(raw_package, index) + package_name = validated_plugins[0][0] + canonical_package_name = canonicalize_name(package_name) + previous_package_name = package_names.get(canonical_package_name) + if previous_package_name is not None: + raise PluginCatalogError( + f"duplicate package name {package_name!r}; canonical name {canonical_package_name!r} " + f"already used by {previous_package_name!r}" + ) + package_names[canonical_package_name] = package_name + + for package_name, plugin_name, entry_point_name in validated_plugins: previous = runtime_names.get(plugin_name) if previous is not None: previous_package, previous_entry_point_name = previous @@ -367,9 +379,8 @@ def _validate_install_metadata(package_name: str, context: str, install: dict[st f"expected a requirement for {package_name!r}" ) - index_url = install.get("index_url") - if index_url is not None: - _catalog_http_url(f"package {package_name!r} install.index_url", index_url) + if "index_url" in install: + _catalog_http_url(f"package {package_name!r} install.index_url", install["index_url"]) def _required_catalog_object( diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py index 5bb87e4dd..a3ec4d051 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py @@ -15,6 +15,7 @@ from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_CATALOG_ALIAS, + DEFAULT_PLUGIN_CATALOG_URL, MAX_PLUGIN_CATALOG_SIZE_BYTES, PLUGIN_CATALOG_CACHE_DIR_NAME, PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, @@ -27,6 +28,7 @@ validate_plugin_catalog_payload, ) from data_designer.cli.repositories.base import ConfigRepository +from data_designer.config.errors import InvalidConfigError, InvalidFileFormatError, InvalidFilePathError from data_designer.config.utils.io_helpers import load_config_file, save_config_file @@ -51,8 +53,8 @@ def load(self) -> PluginCatalogRegistry | None: try: config_dict = load_config_file(self.config_file) return PluginCatalogRegistry.model_validate(config_dict) - except Exception: - return None + except (InvalidConfigError, InvalidFileFormatError, InvalidFilePathError, OSError, ValidationError) as e: + raise PluginCatalogError(f"Failed to load plugin catalog registry at {self.config_file}: {e}") from e def save(self, config: PluginCatalogRegistry) -> None: """Save user-configured plugin catalogs.""" @@ -140,7 +142,7 @@ def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> Pl try: payload = self._fetch_catalog_payload(catalog_config.url) catalog = self._validate_catalog(payload, source=catalog_config.url) - except Exception: + except (PluginCatalogError, OSError, ValueError): if not refresh: cached_catalog = self._load_cached_catalog(catalog_config, require_fresh=False) if cached_catalog is not None: @@ -169,7 +171,7 @@ def _load_cached_catalog(self, catalog: PluginCatalogConfig, *, require_fresh: b return None catalog_payload = cache_payload["catalog"] return self._validate_catalog(catalog_payload, source=str(cache_file)) - except Exception: + except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError): return None def _save_catalog_cache(self, catalog: PluginCatalogConfig, catalog_payload: dict[str, object]) -> None: @@ -199,14 +201,14 @@ def _remove_cache_files(self, catalog: PluginCatalogConfig) -> None: try: with open(cache_file) as f: cache_payload = json.load(f) - except Exception: + except (OSError, json.JSONDecodeError): continue cached_alias = cache_payload.get("catalog_alias") if isinstance(cached_alias, str) and _same_alias(cached_alias, catalog.alias): cache_file.unlink(missing_ok=True) @staticmethod - def _fetch_catalog_payload(location: str) -> dict: + def _fetch_catalog_payload(location: str) -> dict[str, object]: if _is_http_url(location): return _fetch_remote_catalog(location) return _fetch_local_catalog(location) @@ -223,10 +225,11 @@ def _validate_catalog(payload: dict, *, source: str) -> PluginCatalog: @staticmethod def default_catalog() -> PluginCatalogConfig: """Return the built-in NVIDIA plugin catalog configuration.""" + catalog_url = get_default_plugin_catalog_url() return PluginCatalogConfig( alias=DEFAULT_PLUGIN_CATALOG_ALIAS, - url=get_default_plugin_catalog_url(), - trusted=True, + url=catalog_url, + trusted=catalog_url == DEFAULT_PLUGIN_CATALOG_URL, cache_ttl_seconds=PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, ) @@ -239,7 +242,7 @@ def normalize_catalog_location(location: str) -> str: path = Path(location).expanduser() if path.suffix.lower() == ".json": return str(path.resolve(strict=False)) - return str((path / "catalog" / "plugins.json").resolve(strict=False)) + return str(_catalog_plugins_path(path).resolve(strict=False)) def _same_alias(left: str, right: str) -> bool: @@ -262,13 +265,27 @@ def _normalize_catalog_url(url: str) -> str: if len(segments) >= 4 and segments[2] == "tree": ref = segments[3] catalog_root = "/".join(segments[4:]) - catalog_path = f"{catalog_root}/catalog/plugins.json" if catalog_root else "catalog/plugins.json" + catalog_path = _catalog_plugins_url_path(catalog_root) return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{catalog_path}" return url -def _fetch_local_catalog(location: str) -> dict: +def _catalog_plugins_path(path: Path) -> Path: + if path.name == "catalog": + return path / "plugins.json" + return path / "catalog" / "plugins.json" + + +def _catalog_plugins_url_path(catalog_root: str) -> str: + if not catalog_root: + return "catalog/plugins.json" + if catalog_root.rstrip("/").endswith("/catalog") or catalog_root == "catalog": + return f"{catalog_root}/plugins.json" + return f"{catalog_root}/catalog/plugins.json" + + +def _fetch_local_catalog(location: str) -> dict[str, object]: path = Path(location).expanduser() if not path.exists(): raise PluginCatalogError(f"Plugin catalog file not found: {path}") @@ -288,7 +305,7 @@ def _fetch_local_catalog(location: str) -> dict: return payload -def _fetch_remote_catalog(url: str) -> dict: +def _fetch_remote_catalog(url: str) -> dict[str, object]: request = Request(url, headers={"User-Agent": "data-designer"}) try: with urlopen(request, timeout=10) as response: diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index ac2eada53..650476013 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -10,6 +10,8 @@ import sys from collections.abc import Callable +from packaging.utils import canonicalize_name + from data_designer.cli.plugin_catalog import ( PLUGIN_ENTRY_POINT_GROUP, PYPI_SIMPLE_INDEX_URL, @@ -64,18 +66,55 @@ def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: def verify_entry_points(self, entries: list[PluginCatalogEntry]) -> bool: """Verify every declared entry point for an installed catalog package.""" + if not entries: + return False + importlib.invalidate_caches() - installed_entry_point_names = { - entry_point.name for entry_point in importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP) - } - return bool(entries) and all(entry.entry_point.name in installed_entry_point_names for entry in entries) + installed_entry_points = list(importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP)) + return all( + any( + _installed_entry_point_matches(installed_entry_point, entry) + for installed_entry_point in installed_entry_points + ) + for entry in entries + ) def _run_subprocess(command: list[str]) -> int: - result = subprocess.run(command, check=False) + result = subprocess.run(command, check=False, stdin=subprocess.DEVNULL) return result.returncode +def _installed_entry_point_matches( + installed_entry_point: importlib.metadata.EntryPoint, + entry: PluginCatalogEntry, +) -> bool: + if installed_entry_point.name != entry.entry_point.name: + return False + if installed_entry_point.value != entry.entry_point.value: + return False + + distribution_name = _entry_point_distribution_name(installed_entry_point) + if distribution_name is None: + return True + return canonicalize_name(distribution_name) == canonicalize_name(entry.package.name) + + +def _entry_point_distribution_name(installed_entry_point: importlib.metadata.EntryPoint) -> str | None: + distribution = getattr(installed_entry_point, "dist", None) + if distribution is None: + return None + + metadata = getattr(distribution, "metadata", None) + if metadata is None: + return None + + name = metadata.get("Name") + if not isinstance(name, str) or not name: + return None + return name + + def _resolve_manager(manager: str) -> str: if manager not in {"auto", "uv", "pip"}: raise ValueError(f"Unsupported plugin installer {manager!r}. Expected 'auto', 'uv', or 'pip'.") diff --git a/packages/data-designer/tests/cli/commands/test_plugins_command.py b/packages/data-designer/tests/cli/commands/test_plugins_command.py index 4605ce423..83af4c8ea 100644 --- a/packages/data-designer/tests/cli/commands/test_plugins_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugins_command.py @@ -62,6 +62,24 @@ def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMoc ) +def test_plugins_info_help_uses_package_or_runtime_plugin_argument() -> None: + result = runner.invoke(app, ["plugins", "info", "--help"]) + + assert result.exit_code == 0 + assert "PACKAGE_OR_PLUGIN" in result.output + assert "Package name or runtime plugin name" in result.output + + +def test_plugins_install_help_uses_package_first_wording() -> None: + result = runner.invoke(app, ["plugins", "install", "--help"]) + + assert result.exit_code == 0 + assert "PACKAGE_OR_PLUGIN" in result.output + assert "Package name or runtime plugin name" in result.output + assert "Allow installing a catalog" in result.output + assert "package when compatibility" in result.output + + @patch("data_designer.cli.commands.plugins.PluginCatalogController") def test_plugins_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() @@ -103,7 +121,7 @@ def test_plugins_installed_warns_when_parent_catalog_is_unused( assert result.exit_code == 0 mock_print_info.assert_called_once_with( - "Ignoring --catalog 'research'; installed plugins are discovered from the current Python environment." + "Ignoring --catalog 'research'; installed runtime plugins are discovered from the current Python environment." ) mock_ctrl.run_installed.assert_called_once_with() diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 36a96ad94..3c7a98948 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -4,10 +4,11 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest import typer +from rich.table import Table from data_designer.cli.controllers.plugin_catalog_controller import PluginCatalogController from data_designer.cli.plugin_catalog import ( @@ -15,6 +16,7 @@ InstallPlan, PluginCatalogConfig, PluginCatalogEntry, + PluginCatalogError, ) @@ -26,6 +28,150 @@ def controller(tmp_path: Path) -> PluginCatalogController: return plugin_controller +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_list_mentions_hidden_incompatible_packages_when_visible_list_is_empty( + mock_print_warning: MagicMock, + mock_print_info: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.list_entries.side_effect = [[], [entry]] + + controller.run_list(catalog_alias="local") + + assert controller.catalog_service.list_entries.call_args_list == [ + call("local", refresh=False, include_incompatible=False), + call("local", refresh=False, include_incompatible=True), + ] + mock_print_warning.assert_called_once_with("No compatible plugin packages found") + mock_print_info.assert_any_call( + "Incompatible catalog packages are hidden. Use --include-incompatible to show them." + ) + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_search_mentions_hidden_incompatible_packages_when_visible_matches_are_empty( + mock_print_warning: MagicMock, + mock_print_info: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.search_entries.side_effect = [[], [entry]] + + controller.run_search("text", catalog_alias="local") + + assert controller.catalog_service.search_entries.call_args_list == [ + call("text", "local", refresh=False, include_incompatible=False), + call("text", "local", refresh=False, include_incompatible=True), + ] + mock_print_warning.assert_called_once_with("No compatible plugin packages matched") + mock_print_info.assert_any_call( + "Matching incompatible catalog packages are hidden. Use --include-incompatible to show them." + ) + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +def test_run_list_renders_package_first_catalog_table( + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + package_entries = [ + _entry(name="text-column", plugin_type="column-generator"), + _entry(name="text-processor", plugin_type="processor"), + ] + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.list_entries.return_value = package_entries + controller.catalog_service.group_entries_by_package.return_value = { + "data-designer-text-transform": package_entries, + } + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + + controller.run_list(catalog_alias="local", include_incompatible=True) + + printed_tables = [ + call.args[0] for call in mock_console.print.call_args_list if call.args and isinstance(call.args[0], Table) + ] + assert printed_tables + assert printed_tables[0].title == "Catalog Plugin Packages" + assert [column.header for column in printed_tables[0].columns] == [ + "Package", + "Runtime Plugins", + "Compatible", + "Docs", + ] + controller.catalog_service.group_entries_by_package.assert_called_once_with(package_entries) + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.display_config_preview") +def test_run_info_renders_package_metadata_with_nested_runtime_plugins( + mock_display_config_preview: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + package_entries = [ + _entry(name="text-column", plugin_type="column-generator"), + _entry(name="text-processor", plugin_type="processor"), + ] + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_entry.return_value = package_entries[0] + controller.catalog_service.get_package_entries.return_value = package_entries + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = _plan(catalog) + + controller.run_info("data-designer-text-transform", catalog_alias="local") + + metadata = mock_display_config_preview.call_args.args[0] + assert metadata["package"] == { + "name": "data-designer-text-transform", + "description": "Transform text records", + } + assert metadata["install"] == { + "requirement": "data-designer-text-transform", + "index_url": "https://docs.example.test/simple/", + } + assert metadata["plugins"] == [ + { + "name": "text-column", + "plugin_type": "column-generator", + "entry_point": { + "group": "data_designer.plugins", + "name": "text-column", + "value": "data_designer_text_transform.plugin:plugin", + }, + }, + { + "name": "text-processor", + "plugin_type": "processor", + "entry_point": { + "group": "data_designer.plugins", + "name": "text-processor", + "value": "data_designer_text_transform.plugin:plugin", + }, + }, + ] + assert all("package" not in plugin for plugin in metadata["plugins"]) + assert all("install" not in plugin for plugin in metadata["plugins"]) + assert all("compatibility" not in plugin for plugin in metadata["plugins"]) + assert all("docs" not in plugin for plugin in metadata["plugins"]) + mock_display_config_preview.assert_called_once() + assert mock_console.print.call_count >= 1 + + @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") def test_run_install_dry_run_renders_plan_without_installing( @@ -84,10 +230,43 @@ def test_run_install_blocks_incompatible_plugin_without_force( include_incompatible=True, ) controller.install_service.build_install_plan.assert_not_called() - mock_print_error.assert_called_once_with("Plugin 'text-transform' is not compatible with this environment") + mock_print_error.assert_called_once_with( + "Plugin package 'data-designer-text-transform' is not compatible with this environment" + ) mock_console.print.assert_any_call(" - Data Designer 0.5.7 does not satisfy >=99.0") +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_install_dry_run_renders_incompatible_plan_and_block_message( + mock_print_warning: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_entry.return_value = entry + controller.catalog_service.get_package_entries.return_value = [entry] + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( + False, + ["Data Designer 0.5.7 does not satisfy >=99.0"], + ) + controller.install_service.build_install_plan.return_value = _plan(catalog) + + controller.run_install("text-transform", catalog_alias="local", dry_run=True) + + controller.install_service.build_install_plan.assert_called_once_with(entry, catalog, manager="auto") + controller.install_service.install.assert_not_called() + controller.install_service.verify_entry_points.assert_not_called() + mock_console.print.assert_any_call(" Command: [bold]python -m pip install data-designer-text-transform[/bold]") + mock_console.print.assert_any_call(" Compatibility: [bold yellow]not compatible[/bold yellow]") + mock_console.print.assert_any_call(" - Data Designer 0.5.7 does not satisfy >=99.0") + mock_print_warning.assert_called_once_with( + "Dry run complete; no changes made. A real install would be blocked unless you pass --force." + ) + + @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") @@ -219,6 +398,20 @@ def test_run_catalogs_add_wraps_invalid_alias_validation_error( mock_print_error.assert_called_once_with("Invalid catalog alias 'foo/bar': must match `^[A-Za-z0-9_.-]+$`") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +def test_run_catalogs_list_wraps_registry_load_error( + mock_print_error: MagicMock, + controller: PluginCatalogController, +) -> None: + controller.catalog_service.list_catalogs.side_effect = PluginCatalogError("bad registry") + + with pytest.raises(typer.Exit) as exc_info: + controller.run_catalogs_list() + + assert exc_info.value.exit_code == 1 + mock_print_error.assert_called_once_with("Failed to list plugin catalogs: bad registry") + + def _catalog(*, trusted: bool) -> PluginCatalogConfig: return PluginCatalogConfig( alias="local", @@ -239,22 +432,27 @@ def _plan(catalog: PluginCatalogConfig) -> InstallPlan: ) -def _entry() -> PluginCatalogEntry: +def _entry( + *, + name: str = "text-transform", + plugin_type: str = "processor", + package_name: str = "data-designer-text-transform", +) -> PluginCatalogEntry: return PluginCatalogEntry.model_validate( { - "name": "text-transform", - "plugin_type": "processor", + "name": name, + "plugin_type": plugin_type, "description": "Transform text records", "package": { - "name": "data-designer-text-transform", + "name": package_name, }, "install": { - "requirement": "data-designer-text-transform", + "requirement": package_name, "index_url": "https://docs.example.test/simple/", }, "entry_point": { "group": "data_designer.plugins", - "name": "text-transform", + "name": name, "value": "data_designer_text_transform.plugin:plugin", }, "compatibility": { @@ -266,7 +464,7 @@ def _entry() -> PluginCatalogEntry: }, }, "docs": { - "url": "https://docs.example.test/plugins/data-designer-text-transform/", + "url": f"https://docs.example.test/plugins/{package_name}/", }, } ) diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py index 687492159..a0c8c045f 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py @@ -5,12 +5,15 @@ import json from pathlib import Path +from unittest.mock import Mock, patch +from urllib.error import HTTPError import pytest from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_CATALOG_ALIAS, DEFAULT_PLUGIN_CATALOG_URL_ENV_VAR, + MAX_PLUGIN_CATALOG_SIZE_BYTES, PluginCatalogError, ) from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository, normalize_catalog_location @@ -32,6 +35,7 @@ def test_default_catalog_honors_url_environment_override(tmp_path: Path, monkeyp catalog = repository.default_catalog() assert catalog.url == "https://example.test/catalog/plugins.json" + assert catalog.trusted is False def test_add_catalog_normalizes_github_repository_url(tmp_path: Path) -> None: @@ -51,6 +55,14 @@ def test_add_catalog_normalizes_github_tree_url_with_subdirectory(tmp_path: Path assert catalog.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/custom-catalog/catalog/plugins.json" +def test_add_catalog_normalizes_github_tree_url_ending_with_catalog(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + + catalog = repository.add_catalog("research", "https://github.com/acme/dd-plugins/tree/main/catalog") + + assert catalog.url == "https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json" + + def test_catalog_aliases_are_case_insensitive(tmp_path: Path) -> None: repository = PluginCatalogRepository(tmp_path) @@ -73,6 +85,31 @@ def test_normalize_local_catalog_directory() -> None: assert normalized.endswith("/plugins/catalog/plugins.json") +def test_normalize_local_catalog_directory_ending_with_catalog(tmp_path: Path) -> None: + normalized = normalize_catalog_location(str(tmp_path / "plugins" / "catalog")) + + assert normalized == str((tmp_path / "plugins" / "catalog" / "plugins.json").resolve(strict=False)) + + +def test_load_invalid_catalog_registry_raises_user_facing_error(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + repository.config_file.write_text("catalogs:\n- alias: research\n") + + with pytest.raises(PluginCatalogError, match="Failed to load plugin catalog registry"): + repository.load() + + +def test_add_catalog_does_not_replace_invalid_catalog_registry(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + saved_registry = "catalogs:\n- alias: research\n" + repository.config_file.write_text(saved_registry) + + with pytest.raises(PluginCatalogError, match="Failed to load plugin catalog registry"): + repository.add_catalog("local", "https://github.com/acme/dd-plugins") + + assert repository.config_file.read_text() == saved_registry + + def test_load_catalog_uses_cache_when_source_is_unavailable(tmp_path: Path) -> None: catalog_path = _write_catalog(tmp_path) repository = PluginCatalogRepository(tmp_path) @@ -86,6 +123,18 @@ def test_load_catalog_uses_cache_when_source_is_unavailable(tmp_path: Path) -> N assert cached_catalog.plugins[0].name == "text-transform" +def test_load_catalog_falls_back_to_stale_cache_when_refresh_fetch_fails(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path, plugin_name="cached-transform") + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path), cache_ttl_seconds=0) + + repository.load_catalog("local") + catalog_path.unlink() + cached_catalog = repository.load_catalog("local") + + assert cached_catalog.plugins[0].name == "cached-transform" + + def test_load_catalog_with_zero_cache_ttl_refreshes_source(tmp_path: Path) -> None: catalog_path = _write_catalog(tmp_path, plugin_name="text-transform") repository = PluginCatalogRepository(tmp_path) @@ -112,6 +161,42 @@ def test_load_catalog_cache_file_is_keyed_by_alias_and_url(tmp_path: Path) -> No assert cache_files[0].name != "local.json" +@patch("data_designer.cli.repositories.plugin_catalog_repository.urlopen") +def test_load_catalog_reports_remote_http_error(mock_urlopen: Mock, tmp_path: Path) -> None: + mock_urlopen.side_effect = HTTPError( + "https://example.test/catalog/plugins.json", + 404, + "Not Found", + {}, + None, + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("remote", "https://example.test/catalog/plugins.json") + + with pytest.raises(PluginCatalogError, match="HTTP 404"): + repository.load_catalog("remote", refresh=True) + + +@patch("data_designer.cli.repositories.plugin_catalog_repository.urlopen") +def test_load_catalog_rejects_oversized_remote_catalog(mock_urlopen: Mock, tmp_path: Path) -> None: + mock_urlopen.return_value = _RemoteResponse(b"{" + (b" " * MAX_PLUGIN_CATALOG_SIZE_BYTES) + b"}") + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("remote", "https://example.test/catalog/plugins.json") + + with pytest.raises(PluginCatalogError, match="exceeds maximum size"): + repository.load_catalog("remote", refresh=True) + + +@patch("data_designer.cli.repositories.plugin_catalog_repository.urlopen") +def test_load_catalog_reports_remote_json_decode_error(mock_urlopen: Mock, tmp_path: Path) -> None: + mock_urlopen.return_value = _RemoteResponse(b"{") + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("remote", "https://example.test/catalog/plugins.json") + + with pytest.raises(PluginCatalogError, match="Failed to parse plugin catalog JSON"): + repository.load_catalog("remote", refresh=True) + + def test_load_catalog_rejects_unsupported_schema_version(tmp_path: Path) -> None: catalog_path = _write_catalog(tmp_path, schema_version=999) repository = PluginCatalogRepository(tmp_path) @@ -199,6 +284,48 @@ def test_load_catalog_rejects_invalid_schema_v2_install_metadata(tmp_path: Path) repository.load_catalog("local", refresh=True) +def test_load_catalog_rejects_null_schema_v2_install_index_url(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + packages=[ + _package_entry( + package_name="data-designer-invalid-index", + plugins=[_runtime_plugin("invalid-index")], + install={ + "requirement": "data-designer-invalid-index", + "index_url": None, + }, + ) + ], + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="install.index_url.*expected a non-empty string"): + repository.load_catalog("local", refresh=True) + + +def test_load_catalog_rejects_empty_schema_v2_install_index_url(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + packages=[ + _package_entry( + package_name="data-designer-empty-index", + plugins=[_runtime_plugin("empty-index")], + install={ + "requirement": "data-designer-empty-index", + "index_url": "", + }, + ) + ], + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="install.index_url.*expected a non-empty string"): + repository.load_catalog("local", refresh=True) + + def test_load_catalog_rejects_unexpected_schema_v2_fields(tmp_path: Path) -> None: package = _package_entry() package["tags"] = ["extra"] @@ -231,6 +358,43 @@ def test_load_catalog_rejects_duplicate_runtime_plugin_names(tmp_path: Path) -> repository.load_catalog("local", refresh=True) +def test_load_catalog_rejects_duplicate_canonical_package_names(tmp_path: Path) -> None: + catalog_path = _write_catalog( + tmp_path, + packages=[ + _package_entry( + package_name="data-designer-foo", + plugins=[_runtime_plugin("first-plugin")], + ), + _package_entry( + package_name="data_designer_foo", + plugins=[_runtime_plugin("second-plugin")], + ), + ], + ) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + with pytest.raises(PluginCatalogError, match="duplicate package name"): + repository.load_catalog("local", refresh=True) + + +class _RemoteResponse: + def __init__(self, content: bytes, *, status: int = 200) -> None: + self._content = content + self.status = status + + def __enter__(self) -> "_RemoteResponse": + return self + + def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None: + return None + + def read(self, size: int = -1) -> bytes: + _ = size + return self._content + + def _write_catalog( tmp_path: Path, *, diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index 89e581661..9c185df4a 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -59,6 +59,80 @@ def test_evaluate_compatibility_reports_data_designer_constraint(tmp_path: Path) assert result.reasons == ["Data Designer 0.5.7 does not satisfy >=99.0"] +def test_evaluate_compatibility_reports_python_constraint() -> None: + service = PluginCatalogService( + Mock(spec=PluginCatalogRepository), + python_version="3.11.0", + data_designer_version="0.5.7", + ) + entry = PluginCatalogEntry.model_validate( + _entry( + name="future-python-plugin", + plugin_type="processor", + package_name="data-designer-future-python-plugin", + python_specifier=">=3.12", + data_designer_specifier=">=0.5.7", + ) + ) + + result = service.evaluate_compatibility(entry) + + assert result.is_compatible is False + assert result.reasons == ["Python 3.11.0 does not satisfy >=3.12"] + + +@pytest.mark.parametrize( + ("marker", "expected_is_compatible", "expected_reasons"), + [ + ("python_version >= '3.12'", True, []), + ("python_version < '3.12'", False, ["Data Designer 0.5.7 does not satisfy >=99.0"]), + ], +) +def test_evaluate_compatibility_respects_data_designer_marker( + marker: str, + expected_is_compatible: bool, + expected_reasons: list[str], +) -> None: + service = PluginCatalogService( + Mock(spec=PluginCatalogRepository), + python_version="3.11.0", + data_designer_version="0.5.7", + ) + entry = PluginCatalogEntry.model_validate( + _entry( + name="marker-gated-plugin", + plugin_type="processor", + package_name="data-designer-marker-gated-plugin", + data_designer_specifier=">=99.0", + data_designer_marker=marker, + ) + ) + + result = service.evaluate_compatibility(entry) + + assert result.is_compatible is expected_is_compatible + assert result.reasons == expected_reasons + + +@patch("data_designer.cli.services.plugin_catalog_service._get_installed_data_designer_version", return_value=None) +def test_evaluate_compatibility_reports_missing_data_designer_version(mock_version: Mock) -> None: + service = PluginCatalogService(Mock(spec=PluginCatalogRepository), python_version="3.11.0") + entry = PluginCatalogEntry.model_validate( + _entry( + name="compatible-plugin", + plugin_type="processor", + package_name="data-designer-compatible-plugin", + data_designer_specifier=">=0.5.7", + ) + ) + + result = service.evaluate_compatibility(entry) + + assert result.is_compatible is False + assert result.reasons == ["Unable to resolve installed Data Designer version for constraint '>=0.5.7'"] + mock_version.assert_called_once_with() + + def test_evaluate_compatibility_accepts_local_dev_version_above_lower_bound(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService( @@ -212,6 +286,8 @@ def _package( package_name: str, data_designer_specifier: str, plugins: list[dict], + data_designer_marker: str | None = None, + python_specifier: str = ">=3.10", ) -> dict: return { "name": package_name, @@ -221,11 +297,11 @@ def _package( "index_url": "https://docs.example.test/simple/", }, "compatibility": { - "python": {"specifier": ">=3.10"}, + "python": {"specifier": python_specifier}, "data_designer": { "requirement": f"data-designer{data_designer_specifier}", "specifier": data_designer_specifier, - "marker": None, + "marker": data_designer_marker, }, }, "docs": { @@ -253,6 +329,8 @@ def _entry( plugin_type: str, package_name: str, data_designer_specifier: str, + data_designer_marker: str | None = None, + python_specifier: str = ">=3.10", ) -> dict: return { "name": name, @@ -271,11 +349,11 @@ def _entry( "value": f"{package_name.replace('-', '_')}.plugin:plugin", }, "compatibility": { - "python": {"specifier": ">=3.10"}, + "python": {"specifier": python_specifier}, "data_designer": { "requirement": f"data-designer{data_designer_specifier}", "specifier": data_designer_specifier, - "marker": None, + "marker": data_designer_marker, }, }, "docs": { diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index d7eb8e984..e99c86c52 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -57,6 +57,51 @@ def test_build_direct_reference_install_plan_uses_requirement_verbatim() -> None assert "--extra-index-url" not in plan.command +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_chooses_uv_when_available(mock_which: Mock) -> None: + entry = _entry( + package_name="data-designer-template", + install={ + "requirement": "data-designer-template", + "index_url": "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + }, + ) + catalog = PluginCatalogConfig( + alias="nvidia", url="https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json" + ) + service = PluginInstallService() + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.manager == "uv" + assert plan.command == [ + "uv", + "pip", + "install", + "--python", + sys.executable, + "--default-index", + "https://pypi.org/simple/", + "--index", + "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + "data-designer-template", + ] + mock_which.assert_called_once_with("uv") + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value=None) +def test_build_auto_install_plan_chooses_pip_when_uv_is_unavailable(mock_which: Mock) -> None: + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.manager == "pip" + assert plan.command == [sys.executable, "-m", "pip", "install", "data-designer-template"] + mock_which.assert_called_once_with("uv") + + @patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") def test_build_uv_install_plan_targets_current_python_and_adds_catalog_index(mock_which: Mock) -> None: entry = _entry( @@ -87,6 +132,18 @@ def test_build_uv_install_plan_targets_current_python_and_adds_catalog_index(moc ] +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value=None) +def test_build_uv_install_plan_raises_when_uv_is_unavailable(mock_which: Mock) -> None: + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + with pytest.raises(ValueError, match="uv was requested"): + service.build_install_plan(entry, catalog, manager="uv") + + mock_which.assert_called_once_with("uv") + + def test_install_raises_when_runner_fails() -> None: service = PluginInstallService(runner=lambda command: 2) entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) @@ -110,8 +167,8 @@ def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( install={"requirement": "data-designer-template"}, ) mock_entry_points.return_value = [ - SimpleNamespace(name="other-plugin"), - SimpleNamespace(name="text-transform"), + SimpleNamespace(name="other-plugin", value="other_package.plugin:plugin"), + SimpleNamespace(name="text-transform", value="data_designer_template.plugin:plugin"), ] service = PluginInstallService() @@ -120,6 +177,50 @@ def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( mock_entry_points.assert_called_once_with(group="data_designer.plugins") +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") +def test_verify_entry_points_fails_when_name_matches_but_value_differs(mock_entry_points: Mock) -> None: + entry = _entry( + package_name="data-designer-template", + plugin_name="text-transform", + entry_point_name="text-transform", + entry_point_value="data_designer_template.plugin:plugin", + install={"requirement": "data-designer-template"}, + ) + mock_entry_points.return_value = [ + SimpleNamespace(name="text-transform", value="other_package.plugin:plugin"), + ] + service = PluginInstallService() + + assert service.verify_entry_points([entry]) is False + + +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") +def test_verify_entry_points_succeeds_when_all_declared_entries_match(mock_entry_points: Mock) -> None: + entries = [ + _entry( + package_name="data-designer-template", + plugin_name="text-transform", + entry_point_name="text-transform", + entry_point_value="data_designer_template.plugin:plugin", + install={"requirement": "data-designer-template"}, + ), + _entry( + package_name="data-designer-profiler", + plugin_name="text-profiler", + entry_point_name="text-profiler", + entry_point_value="data_designer_profiler.plugin:plugin", + install={"requirement": "data-designer-profiler"}, + ), + ] + mock_entry_points.return_value = [ + SimpleNamespace(name="text-profiler", value="data_designer_profiler.plugin:plugin"), + SimpleNamespace(name="text-transform", value="data_designer_template.plugin:plugin"), + ] + service = PluginInstallService() + + assert service.verify_entry_points(entries) is True + + @patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") def test_verify_entry_points_requires_every_declared_entry_point(mock_entry_points: Mock) -> None: entries = [ @@ -127,27 +228,68 @@ def test_verify_entry_points_requires_every_declared_entry_point(mock_entry_poin package_name="data-designer-retrieval-sdg", plugin_name="document-chunker", entry_point_name="document-chunker", + entry_point_value="data_designer_retrieval_sdg.chunker:plugin", install={"requirement": "data-designer-retrieval-sdg"}, ), _entry( package_name="data-designer-retrieval-sdg", plugin_name="embedding-dedup", entry_point_name="embedding-dedup", + entry_point_value="data_designer_retrieval_sdg.dedup:plugin", install={"requirement": "data-designer-retrieval-sdg"}, ), ] - mock_entry_points.return_value = [SimpleNamespace(name="document-chunker")] + mock_entry_points.return_value = [ + SimpleNamespace(name="document-chunker", value="data_designer_retrieval_sdg.chunker:plugin") + ] service = PluginInstallService() assert service.verify_entry_points(entries) is False +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") +def test_verify_entry_points_verifies_multi_runtime_package_entries(mock_entry_points: Mock) -> None: + entries = [ + _entry( + package_name="data-designer-retrieval-sdg", + plugin_name="document-chunker", + entry_point_name="document-chunker", + entry_point_value="data_designer_retrieval_sdg.chunker:plugin", + install={"requirement": "data-designer-retrieval-sdg"}, + ), + _entry( + package_name="data-designer-retrieval-sdg", + plugin_name="embedding-dedup", + entry_point_name="embedding-dedup", + entry_point_value="data_designer_retrieval_sdg.dedup:plugin", + install={"requirement": "data-designer-retrieval-sdg"}, + ), + ] + distribution = SimpleNamespace(metadata={"Name": "data-designer-retrieval-sdg"}) + mock_entry_points.return_value = [ + SimpleNamespace( + name="embedding-dedup", + value="data_designer_retrieval_sdg.dedup:plugin", + dist=distribution, + ), + SimpleNamespace( + name="document-chunker", + value="data_designer_retrieval_sdg.chunker:plugin", + dist=distribution, + ), + ] + service = PluginInstallService() + + assert service.verify_entry_points(entries) is True + + def _entry( *, package_name: str, install: dict, plugin_name: str = "text-transform", entry_point_name: str = "text-transform", + entry_point_value: str = "data_designer_template.plugin:plugin", ) -> PluginCatalogEntry: payload = { "name": plugin_name, @@ -160,7 +302,7 @@ def _entry( "entry_point": { "group": "data_designer.plugins", "name": entry_point_name, - "value": "data_designer_template.plugin:plugin", + "value": entry_point_value, }, "compatibility": { "python": {"specifier": ">=3.10"}, diff --git a/packages/data-designer/tests/cli/test_main.py b/packages/data-designer/tests/cli/test_main.py index 33b620a35..c40178454 100644 --- a/packages/data-designer/tests/cli/test_main.py +++ b/packages/data-designer/tests/cli/test_main.py @@ -30,6 +30,17 @@ def test_main_bootstraps_before_running_app(mock_bootstrap: Mock, mock_app: Mock assert call_order.mock_calls == [call.bootstrap(), call.app()] +@patch("data_designer.cli.main.app") +@patch("data_designer.cli.main.ensure_cli_default_model_settings") +def test_main_bootstraps_for_plugins_commands(mock_bootstrap: Mock, mock_app: Mock) -> None: + """Plugin commands still run through CLI default setup before Typer dispatch.""" + with patch("sys.argv", ["data-designer", "plugins", "list"]): + main() + + mock_bootstrap.assert_called_once_with() + mock_app.assert_called_once_with() + + @patch("data_designer.cli.main.app") @patch("data_designer.cli.main.ensure_cli_default_model_settings") def test_main_skips_bootstrap_for_version(mock_bootstrap: Mock, mock_app: Mock) -> None: @@ -167,3 +178,47 @@ def test_app_dispatches_lazy_create_command(mock_controller_cls: Mock) -> None: resume=ResumeMode.NEVER, output_format=None, ) + + +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_app_dispatches_lazy_plugins_list_command(mock_controller_cls: Mock) -> None: + """The plugins group lazily resolves command callbacks without loading a catalog.""" + mock_controller = Mock() + mock_controller_cls.return_value = mock_controller + + result = runner.invoke( + app, + ["plugins", "--catalog", "local", "list", "--refresh", "--include-incompatible"], + ) + + assert result.exit_code == 0 + mock_controller.run_list.assert_called_once_with( + catalog_alias="local", + refresh=True, + include_incompatible=True, + ) + + +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_app_dispatches_lazy_plugin_catalogs_list_command(mock_controller_cls: Mock) -> None: + """Nested plugin catalog commands resolve through the lazy command group.""" + mock_controller = Mock() + mock_controller_cls.return_value = mock_controller + + result = runner.invoke(app, ["plugins", "catalogs", "list"]) + + assert result.exit_code == 0 + mock_controller.run_catalogs_list.assert_called_once_with() + + +def test_app_help_keeps_config_and_plugins_commands_reachable() -> None: + config_result = runner.invoke(app, ["config", "--help"]) + plugins_result = runner.invoke(app, ["plugins", "--help"]) + + assert config_result.exit_code == 0 + assert "providers" in config_result.output + assert "models" in config_result.output + assert plugins_result.exit_code == 0 + assert "list" in plugins_result.output + assert "install" in plugins_result.output + assert "catalogs" in plugins_result.output From 6adf2cffe86ab5c36952a0ef48748718d91aeeb9 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 20:49:43 +0000 Subject: [PATCH 14/34] add plugin package uninstall workflow --- .../src/data_designer/cli/commands/plugins.py | 64 +++++++++-- .../controllers/plugin_catalog_controller.py | 105 ++++++++++++------ .../src/data_designer/cli/main.py | 9 +- .../src/data_designer/cli/plugin_catalog.py | 12 +- .../cli/services/plugin_catalog_service.py | 32 ++++-- .../cli/services/plugin_install_service.py | 55 ++++++++- 6 files changed, 217 insertions(+), 60 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugins.py index fae7adb1f..6f3bc9405 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugins.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugins.py @@ -71,9 +71,9 @@ def search_command( def info_command( ctx: typer.Context, - package_or_plugin: str = typer.Argument( - help="Package name or runtime plugin name from the catalog.", - metavar="PACKAGE_OR_PLUGIN", + package: str = typer.Argument( + help="Plugin package name or package alias from the catalog.", + metavar="PACKAGE", ), catalog: str | None = typer.Option( None, @@ -89,7 +89,7 @@ def info_command( """Show metadata, compatibility, docs, and install plan for one plugin package.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_info( - package_or_plugin, + package, catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, ) @@ -97,9 +97,9 @@ def info_command( def install_command( ctx: typer.Context, - package_or_plugin: str = typer.Argument( - help="Package name or runtime plugin name from the catalog.", - metavar="PACKAGE_OR_PLUGIN", + package: str = typer.Argument( + help="Plugin package name or package alias from the catalog.", + metavar="PACKAGE", ), catalog: str | None = typer.Option( None, @@ -134,10 +134,10 @@ def install_command( help="Allow installing a catalog package when compatibility checks fail.", ), ) -> None: - """Install one Data Designer plugin package, then verify runtime discovery.""" + """Install one Data Designer plugin package, then verify package registration.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_install( - package_or_plugin, + package, catalog_alias=_resolve_catalog_alias(ctx, catalog), refresh=refresh, manager=manager, @@ -147,6 +147,52 @@ def install_command( ) +def uninstall_command( + ctx: typer.Context, + package: str = typer.Argument( + help="Plugin package name or package alias from the catalog.", + metavar="PACKAGE", + ), + catalog: str | None = typer.Option( + None, + "--catalog", + help="Plugin catalog alias to uninstall from. Can also be provided before the subcommand.", + ), + refresh: bool = typer.Option( + False, + "--refresh", + help="Fetch the catalog even when a fresh cache entry exists.", + ), + manager: str = typer.Option( + "auto", + "--manager", + click_type=click.Choice(["auto", "uv", "pip"]), + help="Package manager to use for uninstallation.", + ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Uninstall without an interactive confirmation prompt.", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Print the uninstall plan without mutating the current environment.", + ), +) -> None: + """Uninstall one Data Designer plugin package, then verify package registration is removed.""" + controller = PluginCatalogController(DATA_DESIGNER_HOME) + controller.run_uninstall( + package, + catalog_alias=_resolve_catalog_alias(ctx, catalog), + refresh=refresh, + manager=manager, + yes=yes, + dry_run=dry_run, + ) + + def installed_command(ctx: typer.Context) -> None: """List installed Data Designer runtime plugin entry points.""" _warn_if_parent_catalog_unused(ctx, "installed runtime plugins are discovered from the current Python environment") diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 943737fa4..d3b0f35e4 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -36,7 +36,7 @@ class PluginCatalogController: - """Controller for plugin catalog browsing, alias management, and install workflows. + """Controller for plugin catalog browsing, alias management, and package workflows. Catalog browsing and environment mutation intentionally use separate services so read-only catalog operations stay decoupled from package-manager execution. @@ -103,20 +103,20 @@ def run_search( def run_info( self, - package_or_plugin: str, + package_name: str, *, catalog_alias: str | None = None, refresh: bool = False, ) -> None: """Show full metadata for one plugin package.""" catalog = self._get_catalog_or_exit(catalog_alias) - entry = self._get_entry_or_exit(package_or_plugin, catalog.alias, refresh=refresh) package_entries = self._get_package_entries_or_exit( - entry.package.name, + package_name, catalog.alias, refresh=refresh, include_incompatible=True, - ) or [entry] + ) + entry = package_entries[0] compatibility = self.catalog_service.evaluate_compatibility(entry) print_header(f"Plugin Package: {entry.package.name}") @@ -154,7 +154,7 @@ def run_info( def run_install( self, - package_or_plugin: str, + package_name: str, *, catalog_alias: str | None = None, refresh: bool = False, @@ -165,13 +165,13 @@ def run_install( ) -> None: """Install one plugin package from the catalog.""" catalog = self._get_catalog_or_exit(catalog_alias) - entry = self._get_entry_or_exit(package_or_plugin, catalog.alias, refresh=refresh, include_incompatible=True) package_entries = self._get_package_entries_or_exit( - entry.package.name, + package_name, catalog.alias, refresh=refresh, include_incompatible=True, - ) or [entry] + ) + entry = package_entries[0] compatibility = self.catalog_service.evaluate_compatibility(entry) if not compatibility.is_compatible and not force and not dry_run: @@ -188,7 +188,6 @@ def run_install( print_header("Install Data Designer Plugin Package") console.print(f" Package: [bold]{entry.package.name}[/bold]") - console.print(f" Runtime plugins: [bold]{_format_runtime_plugins(package_entries)}[/bold]") console.print(f" Catalog: [bold]{catalog.alias}[/bold] ({catalog.url})") console.print(f" Requirement: [bold]{entry.install.requirement}[/bold]") if entry.install.index_url is not None: @@ -198,7 +197,8 @@ def run_install( if not catalog.trusted: print_warning( - "This catalog is not marked trusted. Plugin installation executes Python package code from the requirement above." + "This catalog is not marked trusted. Plugin package installation executes Python package code from " + "the requirement above." ) if dry_run: @@ -221,12 +221,64 @@ def run_install( raise typer.Exit(code=1) if self.install_service.verify_entry_points(package_entries): - print_success(f"Plugin package {entry.package.name!r} installed and discovered") + print_success(f"Plugin package {entry.package.name!r} installed and registered") else: print_warning( f"Plugin package {entry.package.name!r} was installed, but Data Designer did not discover every " - "declared entry point. " - "Restart the shell or check the package entry point metadata." + "declared package entry point. Restart the shell or check the package entry point metadata." + ) + + def run_uninstall( + self, + package_name: str, + *, + catalog_alias: str | None = None, + refresh: bool = False, + manager: str = "auto", + yes: bool = False, + dry_run: bool = False, + ) -> None: + """Uninstall one plugin package resolved from the catalog.""" + catalog = self._get_catalog_or_exit(catalog_alias) + package_entries = self._get_package_entries_or_exit( + package_name, + catalog.alias, + refresh=refresh, + include_incompatible=True, + ) + entry = package_entries[0] + + try: + plan = self.install_service.build_uninstall_plan(entry, catalog, manager=manager) + except ValueError as e: + print_error(f"Failed to build plugin uninstall plan: {e}") + raise typer.Exit(code=1) + + print_header("Uninstall Data Designer Plugin Package") + console.print(f" Package: [bold]{entry.package.name}[/bold]") + console.print(f" Catalog: [bold]{catalog.alias}[/bold] ({catalog.url})") + console.print(f" Command: [bold]{shlex.join(plan.command)}[/bold]") + + if dry_run: + print_info("Dry run complete; no changes made") + return + + if not yes and not confirm_action("Uninstall this package from the current Python environment?", default=False): + print_info("No changes made") + return + + try: + self.install_service.uninstall(plan) + except RuntimeError as e: + print_error(str(e)) + raise typer.Exit(code=1) + + if self.install_service.verify_entry_points_removed(package_entries): + print_success(f"Plugin package {entry.package.name!r} uninstalled and no longer registered") + else: + print_warning( + f"Plugin package {entry.package.name!r} was uninstalled, but Data Designer still discovers one or " + "more declared package entry points. Restart the shell or check the package environment." ) def run_installed(self) -> None: @@ -343,25 +395,6 @@ def _search_entries_or_exit( print_error(f"Failed to search plugin catalog: {e}") raise typer.Exit(code=1) - def _get_entry_or_exit( - self, - package_or_plugin: str, - catalog_alias: str, - *, - refresh: bool, - include_incompatible: bool = True, - ) -> PluginCatalogEntry: - try: - return self.catalog_service.get_entry( - package_or_plugin, - catalog_alias, - refresh=refresh, - include_incompatible=include_incompatible, - ) - except (PluginCatalogError, OSError, ValueError) as e: - print_error(str(e)) - raise typer.Exit(code=1) - def _get_package_entries_or_exit( self, package_name: str, @@ -371,7 +404,7 @@ def _get_package_entries_or_exit( include_incompatible: bool, ) -> list[PluginCatalogEntry]: try: - return self.catalog_service.get_package_entries( + package_entries = self.catalog_service.get_package_entries( package_name, catalog_alias, refresh=refresh, @@ -380,6 +413,10 @@ def _get_package_entries_or_exit( except (PluginCatalogError, OSError, ValueError) as e: print_error(f"Failed to load plugin package metadata: {e}") raise typer.Exit(code=1) + if not package_entries: + print_error(f"Plugin package or alias {package_name!r} was not found in catalog {catalog_alias!r}") + raise typer.Exit(code=1) + return package_entries def _display_empty_list_state(self, catalog_alias: str, *, include_incompatible: bool) -> None: if include_incompatible: diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index 150f7f634..77adc493d 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -144,7 +144,7 @@ def _is_version_request(args: list[str]) -> bool: # Create plugins command group plugins_app = typer.Typer( name="plugins", - help="Discover and install Data Designer plugins from catalogs", + help="Discover, install, and uninstall Data Designer plugin packages from catalogs", cls=create_lazy_typer_group( { "list": { @@ -165,7 +165,12 @@ def _is_version_request(args: list[str]) -> bool: "install": { "module": f"{_CMD}.plugins", "attr": "install_command", - "help": "Install a plugin package and verify runtime discovery", + "help": "Install a plugin package and verify package registration", + }, + "uninstall": { + "module": f"{_CMD}.plugins", + "attr": "uninstall_command", + "help": "Uninstall a plugin package and verify package registration is removed", }, "installed": { "module": f"{_CMD}.plugins", diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index 36ee0cfed..ab46a97fc 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -24,6 +24,7 @@ PLUGIN_CATALOG_SCHEMA_VERSION = 2 PLUGIN_CATALOG_ALIAS_PATTERN = r"^[A-Za-z0-9_.-]+$" DATA_DESIGNER_DISTRIBUTION_NAME = "data-designer" +DATA_DESIGNER_PLUGIN_PACKAGE_PREFIX = "data-designer-" PLUGIN_ENTRY_POINT_GROUP = "data_designer.plugins" PYPI_SIMPLE_INDEX_URL = "https://pypi.org/simple/" CATALOG_DOCUMENT_KEYS = {"packages", "schema_version"} @@ -205,7 +206,6 @@ class CompatibilityResult: class InstallPlan: """Resolved package-manager command for installing one plugin package.""" - plugin_name: str package_name: str source_description: str command: list[str] @@ -214,6 +214,16 @@ class InstallPlan: trusted_catalog: bool +@dataclass(frozen=True) +class UninstallPlan: + """Resolved package-manager command for uninstalling one plugin package.""" + + package_name: str + command: list[str] + manager: str + catalog_alias: str + + @dataclass(frozen=True) class InstalledPluginInfo: """Installed plugin entry point discovered without importing plugin code.""" diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index e7cd0ffc1..0c0fb5ed2 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -14,6 +14,7 @@ from packaging.version import InvalidVersion, Version from data_designer.cli.plugin_catalog import ( + DATA_DESIGNER_PLUGIN_PACKAGE_PREFIX, DEFAULT_PLUGIN_CATALOG_ALIAS, PLUGIN_ENTRY_POINT_GROUP, CompatibilityResult, @@ -103,22 +104,25 @@ def get_entry( def get_package_entries( self, - package_name: str, + package: str, catalog_alias: str | None = None, *, refresh: bool = False, include_incompatible: bool = True, ) -> list[PluginCatalogEntry]: - """Return all runtime plugin entries declared by one catalog package.""" - canonical_package_name = canonicalize_name(package_name) + """Return all runtime plugin entries declared by one catalog package name or package alias.""" + entries = self.list_entries( + catalog_alias, + refresh=refresh, + include_incompatible=include_incompatible, + ) + canonical_package = canonicalize_name(package) + exact_matches = [entry for entry in entries if canonicalize_name(entry.package.name) == canonical_package] + if exact_matches: + return exact_matches + return [ - entry - for entry in self.list_entries( - catalog_alias, - refresh=refresh, - include_incompatible=include_incompatible, - ) - if canonicalize_name(entry.package.name) == canonical_package_name + entry for entry in entries if _package_alias(canonicalize_name(entry.package.name)) == canonical_package ] @staticmethod @@ -243,11 +247,13 @@ def _tokenize(value: str) -> list[str]: def _entry_search_text(entry: PluginCatalogEntry) -> str: + package_name = canonicalize_name(entry.package.name) values = [ entry.name, entry.plugin_type.value, entry.description, entry.package.name, + _package_alias(package_name) or "", entry.install.requirement, entry.install.index_url or "", entry.entry_point.name, @@ -257,6 +263,12 @@ def _entry_search_text(entry: PluginCatalogEntry) -> str: return " ".join(values).lower() +def _package_alias(canonical_package_name: str) -> str | None: + if not canonical_package_name.startswith(DATA_DESIGNER_PLUGIN_PACKAGE_PREFIX): + return None + return canonical_package_name.removeprefix(DATA_DESIGNER_PLUGIN_PACKAGE_PREFIX) + + def _major_minor(version: str) -> str: parts = version.split(".") if len(parts) < 2: diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 650476013..8cce6bbf6 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -18,13 +18,14 @@ InstallPlan, PluginCatalogConfig, PluginCatalogEntry, + UninstallPlan, ) InstallRunner = Callable[[list[str]], int] class PluginInstallService: - """Resolve, execute, and verify plugin installation plans.""" + """Resolve, execute, and verify plugin package install/uninstall plans.""" def __init__(self, runner: InstallRunner | None = None) -> None: self._runner = runner or _run_subprocess @@ -41,7 +42,6 @@ def build_install_plan( install_args, source_description = _install_args_for_entry(entry, resolved_manager) command = _base_command(resolved_manager) + install_args return InstallPlan( - plugin_name=entry.name, package_name=entry.package.name, source_description=source_description, command=command, @@ -50,6 +50,22 @@ def build_install_plan( trusted_catalog=catalog.trusted, ) + def build_uninstall_plan( + self, + entry: PluginCatalogEntry, + catalog: PluginCatalogConfig, + *, + manager: str = "auto", + ) -> UninstallPlan: + """Build the exact package-manager command to uninstall one catalog package.""" + resolved_manager = _resolve_manager(manager) + return UninstallPlan( + package_name=entry.package.name, + command=_base_uninstall_command(resolved_manager) + [entry.package.name], + manager=resolved_manager, + catalog_alias=catalog.alias, + ) + def install(self, plan: InstallPlan) -> None: """Run an installation plan. @@ -58,7 +74,17 @@ def install(self, plan: InstallPlan) -> None: """ return_code = self._runner(plan.command) if return_code != 0: - raise RuntimeError(f"Plugin installer exited with status {return_code}") + raise RuntimeError(f"Plugin package installer exited with status {return_code}") + + def uninstall(self, plan: UninstallPlan) -> None: + """Run an uninstall plan. + + Raises: + RuntimeError: If the package manager exits unsuccessfully. + """ + return_code = self._runner(plan.command) + if return_code != 0: + raise RuntimeError(f"Plugin package uninstaller exited with status {return_code}") def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: """Verify the plugin's declared entry point is installed.""" @@ -79,6 +105,21 @@ def verify_entry_points(self, entries: list[PluginCatalogEntry]) -> bool: for entry in entries ) + def verify_entry_points_removed(self, entries: list[PluginCatalogEntry]) -> bool: + """Verify every declared entry point for a catalog package is no longer installed.""" + if not entries: + return False + + importlib.invalidate_caches() + installed_entry_points = list(importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP)) + return all( + not any( + _installed_entry_point_matches(installed_entry_point, entry) + for installed_entry_point in installed_entry_points + ) + for entry in entries + ) + def _run_subprocess(command: list[str]) -> int: result = subprocess.run(command, check=False, stdin=subprocess.DEVNULL) @@ -121,7 +162,7 @@ def _resolve_manager(manager: str) -> str: if manager == "auto": return "uv" if shutil.which("uv") else "pip" if manager == "uv" and not shutil.which("uv"): - raise ValueError("uv was requested for plugin installation, but it is not available on PATH") + raise ValueError("uv was requested for plugin package installation, but it is not available on PATH") return manager @@ -131,6 +172,12 @@ def _base_command(manager: str) -> list[str]: return [sys.executable, "-m", "pip", "install"] +def _base_uninstall_command(manager: str) -> list[str]: + if manager == "uv": + return ["uv", "pip", "uninstall", "--python", sys.executable] + return [sys.executable, "-m", "pip", "uninstall", "--yes"] + + def _install_args_for_entry(entry: PluginCatalogEntry, manager: str) -> tuple[list[str], str]: requirement = entry.install.requirement index_url = entry.install.index_url From 208948b67e2130e70b5c41f763f46efa95e7a2b0 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 20:49:46 +0000 Subject: [PATCH 15/34] test plugin package command targets --- .../cli/commands/test_plugins_command.py | 50 +++- .../test_plugin_catalog_controller.py | 218 +++++++++++++++--- .../services/test_plugin_catalog_service.py | 71 ++++++ .../services/test_plugin_install_service.py | 93 ++++++++ packages/data-designer/tests/cli/test_main.py | 1 + 5 files changed, 398 insertions(+), 35 deletions(-) diff --git a/packages/data-designer/tests/cli/commands/test_plugins_command.py b/packages/data-designer/tests/cli/commands/test_plugins_command.py index 83af4c8ea..2fffd17f2 100644 --- a/packages/data-designer/tests/cli/commands/test_plugins_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugins_command.py @@ -48,11 +48,14 @@ def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMoc mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "install", "text-transform", "--manager", "pip", "--yes", "--dry-run"]) + result = runner.invoke( + app, + ["plugins", "install", "data-designer-text-transform", "--manager", "pip", "--yes", "--dry-run"], + ) assert result.exit_code == 0 mock_ctrl.run_install.assert_called_once_with( - "text-transform", + "data-designer-text-transform", catalog_alias=None, refresh=False, manager="pip", @@ -62,24 +65,57 @@ def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMoc ) -def test_plugins_info_help_uses_package_or_runtime_plugin_argument() -> None: +@patch("data_designer.cli.commands.plugins.PluginCatalogController") +def test_plugins_uninstall_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: + mock_ctrl = MagicMock() + mock_ctrl_cls.return_value = mock_ctrl + + result = runner.invoke( + app, + ["plugins", "uninstall", "data-designer-text-transform", "--manager", "pip", "--yes", "--dry-run"], + ) + + assert result.exit_code == 0 + mock_ctrl.run_uninstall.assert_called_once_with( + "data-designer-text-transform", + catalog_alias=None, + refresh=False, + manager="pip", + yes=True, + dry_run=True, + ) + + +def test_plugins_info_help_uses_package_argument() -> None: result = runner.invoke(app, ["plugins", "info", "--help"]) assert result.exit_code == 0 - assert "PACKAGE_OR_PLUGIN" in result.output - assert "Package name or runtime plugin name" in result.output + assert "PACKAGE" in result.output + assert "Plugin package name or package alias" in result.output + assert "runtime plugin name" not in result.output def test_plugins_install_help_uses_package_first_wording() -> None: result = runner.invoke(app, ["plugins", "install", "--help"]) assert result.exit_code == 0 - assert "PACKAGE_OR_PLUGIN" in result.output - assert "Package name or runtime plugin name" in result.output + assert "PACKAGE" in result.output + assert "Plugin package name or package alias" in result.output + assert "runtime plugin name" not in result.output assert "Allow installing a catalog" in result.output assert "package when compatibility" in result.output +def test_plugins_uninstall_help_uses_package_first_wording() -> None: + result = runner.invoke(app, ["plugins", "uninstall", "--help"]) + + assert result.exit_code == 0 + assert "PACKAGE" in result.output + assert "Plugin package name or package alias" in result.output + assert "runtime plugin name" not in result.output + assert "Print the uninstall plan" in result.output + + @patch("data_designer.cli.commands.plugins.PluginCatalogController") def test_plugins_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 3c7a98948..d129eff5e 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -17,6 +17,7 @@ PluginCatalogConfig, PluginCatalogEntry, PluginCatalogError, + UninstallPlan, ) @@ -128,12 +129,11 @@ def test_run_info_renders_package_metadata_with_nested_runtime_plugins( ] catalog = _catalog(trusted=True) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = package_entries[0] controller.catalog_service.get_package_entries.return_value = package_entries controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_info("data-designer-text-transform", catalog_alias="local") + controller.run_info("text-transform", catalog_alias="local") metadata = mock_display_config_preview.call_args.args[0] assert metadata["package"] == { @@ -168,10 +168,40 @@ def test_run_info_renders_package_metadata_with_nested_runtime_plugins( assert all("install" not in plugin for plugin in metadata["plugins"]) assert all("compatibility" not in plugin for plugin in metadata["plugins"]) assert all("docs" not in plugin for plugin in metadata["plugins"]) + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "text-transform", + "local", + refresh=False, + include_incompatible=True, + ) mock_display_config_preview.assert_called_once() assert mock_console.print.call_count >= 1 +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +def test_run_info_rejects_runtime_plugin_name_that_is_not_package_alias( + mock_print_error: MagicMock, + controller: PluginCatalogController, +) -> None: + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [] + + with pytest.raises(typer.Exit) as exc_info: + controller.run_info("text-column", catalog_alias="local") + + assert exc_info.value.exit_code == 1 + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "text-column", + "local", + refresh=False, + include_incompatible=True, + ) + mock_print_error.assert_called_once_with("Plugin package or alias 'text-column' was not found in catalog 'local'") + + @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") def test_run_install_dry_run_renders_plan_without_installing( @@ -183,15 +213,15 @@ def test_run_install_dry_run_renders_plan_without_installing( catalog = _catalog(trusted=True) plan = _plan(catalog) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = plan - controller.run_install("text-transform", catalog_alias="local", dry_run=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) - controller.catalog_service.get_entry.assert_called_once_with( - "text-transform", + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "data-designer-text-transform", "local", refresh=False, include_incompatible=True, @@ -199,12 +229,13 @@ def test_run_install_dry_run_renders_plan_without_installing( controller.install_service.install.assert_not_called() controller.install_service.verify_entry_points.assert_not_called() mock_print_info.assert_any_call("Dry run complete; no changes made") + assert all("Runtime plugins" not in str(call_args.args[0]) for call_args in mock_console.print.call_args_list) assert mock_console.print.call_count >= 1 @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") -def test_run_install_blocks_incompatible_plugin_without_force( +def test_run_install_blocks_incompatible_package_without_force( mock_print_error: MagicMock, mock_console: MagicMock, controller: PluginCatalogController, @@ -212,7 +243,6 @@ def test_run_install_blocks_incompatible_plugin_without_force( entry = _entry() catalog = _catalog(trusted=True) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( False, @@ -220,11 +250,12 @@ def test_run_install_blocks_incompatible_plugin_without_force( ) with pytest.raises(typer.Exit) as exc_info: - controller.run_install("text-transform", catalog_alias="local") + controller.run_install("data-designer-text-transform", catalog_alias="local") assert exc_info.value.exit_code == 1 - controller.catalog_service.get_entry.assert_called_once_with( - "text-transform", + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "data-designer-text-transform", "local", refresh=False, include_incompatible=True, @@ -236,6 +267,30 @@ def test_run_install_blocks_incompatible_plugin_without_force( mock_console.print.assert_any_call(" - Data Designer 0.5.7 does not satisfy >=99.0") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +def test_run_install_rejects_runtime_plugin_name_as_target( + mock_print_error: MagicMock, + controller: PluginCatalogController, +) -> None: + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [] + + with pytest.raises(typer.Exit) as exc_info: + controller.run_install("text-column", catalog_alias="local") + + assert exc_info.value.exit_code == 1 + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "text-column", + "local", + refresh=False, + include_incompatible=True, + ) + controller.install_service.build_install_plan.assert_not_called() + mock_print_error.assert_called_once_with("Plugin package or alias 'text-column' was not found in catalog 'local'") + + @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") def test_run_install_dry_run_renders_incompatible_plan_and_block_message( @@ -246,7 +301,6 @@ def test_run_install_dry_run_renders_incompatible_plan_and_block_message( entry = _entry() catalog = _catalog(trusted=True) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( False, @@ -254,7 +308,7 @@ def test_run_install_dry_run_renders_incompatible_plan_and_block_message( ) controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_install("text-transform", catalog_alias="local", dry_run=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) controller.install_service.build_install_plan.assert_called_once_with(entry, catalog, manager="auto") controller.install_service.install.assert_not_called() @@ -279,7 +333,6 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( entry = _entry() catalog = _catalog(trusted=True) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( False, @@ -287,10 +340,11 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( ) controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_install("text-transform", catalog_alias="local", dry_run=True, force=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True, force=True) - controller.catalog_service.get_entry.assert_called_once_with( - "text-transform", + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "data-designer-text-transform", "local", refresh=False, include_incompatible=True, @@ -312,15 +366,15 @@ def test_run_install_warns_for_untrusted_catalog( entry = _entry() catalog = _catalog(trusted=False) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_install("text-transform", catalog_alias="local", dry_run=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) mock_print_warning.assert_called_once_with( - "This catalog is not marked trusted. Plugin installation executes Python package code from the requirement above." + "This catalog is not marked trusted. Plugin package installation executes Python package code from " + "the requirement above." ) assert mock_console.print.call_count >= 1 @@ -336,17 +390,16 @@ def test_run_install_reports_success_when_verification_finds_entry_point( catalog = _catalog(trusted=True) plan = _plan(catalog) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = plan controller.install_service.verify_entry_points.return_value = True - controller.run_install("text-transform", catalog_alias="local", yes=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", yes=True) controller.install_service.install.assert_called_once_with(plan) controller.install_service.verify_entry_points.assert_called_once_with([entry]) - mock_print_success.assert_called_once_with("Plugin package 'data-designer-text-transform' installed and discovered") + mock_print_success.assert_called_once_with("Plugin package 'data-designer-text-transform' installed and registered") assert mock_console.print.call_count >= 1 @@ -361,20 +414,121 @@ def test_run_install_warns_when_verification_misses_entry_point( catalog = _catalog(trusted=True) plan = _plan(catalog) controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_entry.return_value = entry controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) controller.install_service.build_install_plan.return_value = plan controller.install_service.verify_entry_points.return_value = False - controller.run_install("text-transform", catalog_alias="local", yes=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", yes=True) controller.install_service.install.assert_called_once_with(plan) controller.install_service.verify_entry_points.assert_called_once_with([entry]) mock_print_warning.assert_called_once_with( "Plugin package 'data-designer-text-transform' was installed, but Data Designer did not discover every " - "declared entry point. " - "Restart the shell or check the package entry point metadata." + "declared package entry point. Restart the shell or check the package entry point metadata." + ) + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") +def test_run_uninstall_dry_run_renders_plan_without_uninstalling( + mock_print_info: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + plan = _uninstall_plan(catalog) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [entry] + controller.install_service.build_uninstall_plan.return_value = plan + + controller.run_uninstall("data-designer-text-transform", catalog_alias="local", dry_run=True) + + controller.catalog_service.get_entry.assert_not_called() + controller.catalog_service.get_package_entries.assert_called_once_with( + "data-designer-text-transform", + "local", + refresh=False, + include_incompatible=True, + ) + controller.install_service.build_uninstall_plan.assert_called_once_with(entry, catalog, manager="auto") + controller.install_service.uninstall.assert_not_called() + controller.install_service.verify_entry_points_removed.assert_not_called() + mock_console.print.assert_any_call( + " Command: [bold]python -m pip uninstall --yes data-designer-text-transform[/bold]" + ) + assert all("Runtime plugins" not in str(call_args.args[0]) for call_args in mock_console.print.call_args_list) + mock_print_info.assert_any_call("Dry run complete; no changes made") + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") +def test_run_uninstall_wraps_plan_error( + mock_print_error: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [entry] + controller.install_service.build_uninstall_plan.side_effect = ValueError("uv was requested") + + with pytest.raises(typer.Exit) as exc_info: + controller.run_uninstall("data-designer-text-transform", catalog_alias="local") + + assert exc_info.value.exit_code == 1 + controller.install_service.uninstall.assert_not_called() + mock_print_error.assert_called_once_with("Failed to build plugin uninstall plan: uv was requested") + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_success") +def test_run_uninstall_reports_success_when_entry_points_are_removed( + mock_print_success: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + plan = _uninstall_plan(catalog) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [entry] + controller.install_service.build_uninstall_plan.return_value = plan + controller.install_service.verify_entry_points_removed.return_value = True + + controller.run_uninstall("data-designer-text-transform", catalog_alias="local", yes=True) + + controller.install_service.uninstall.assert_called_once_with(plan) + controller.install_service.verify_entry_points_removed.assert_called_once_with([entry]) + mock_print_success.assert_called_once_with( + "Plugin package 'data-designer-text-transform' uninstalled and no longer registered" + ) + assert mock_console.print.call_count >= 1 + + +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_uninstall_warns_when_entry_points_remain( + mock_print_warning: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + plan = _uninstall_plan(catalog) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [entry] + controller.install_service.build_uninstall_plan.return_value = plan + controller.install_service.verify_entry_points_removed.return_value = False + + controller.run_uninstall("data-designer-text-transform", catalog_alias="local", yes=True) + + controller.install_service.uninstall.assert_called_once_with(plan) + controller.install_service.verify_entry_points_removed.assert_called_once_with([entry]) + mock_print_warning.assert_called_once_with( + "Plugin package 'data-designer-text-transform' was uninstalled, but Data Designer still discovers one or " + "more declared package entry points. Restart the shell or check the package environment." ) assert mock_console.print.call_count >= 1 @@ -422,7 +576,6 @@ def _catalog(*, trusted: bool) -> PluginCatalogConfig: def _plan(catalog: PluginCatalogConfig) -> InstallPlan: return InstallPlan( - plugin_name="text-transform", package_name="data-designer-text-transform", source_description="data-designer-text-transform", command=["python", "-m", "pip", "install", "data-designer-text-transform"], @@ -432,6 +585,15 @@ def _plan(catalog: PluginCatalogConfig) -> InstallPlan: ) +def _uninstall_plan(catalog: PluginCatalogConfig) -> UninstallPlan: + return UninstallPlan( + package_name="data-designer-text-transform", + command=["python", "-m", "pip", "uninstall", "--yes", "data-designer-text-transform"], + manager="pip", + catalog_alias=catalog.alias, + ) + + def _entry( *, name: str = "text-transform", diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index 9c185df4a..df93051c9 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -181,6 +181,77 @@ def test_get_entry_resolves_package_name() -> None: assert entry.package.name == "data-designer-package-target" +def test_get_package_entries_resolves_package_alias() -> None: + repository = Mock(spec=PluginCatalogRepository) + repository.load_catalog.return_value = PluginCatalog.model_validate( + { + "schema_version": 2, + "packages": [ + _package( + package_name="data-designer-calculator", + data_designer_specifier=">=0.5.7", + plugins=[ + _runtime_plugin(name="arithmetic-column", plugin_type="column-generator"), + _runtime_plugin(name="arithmetic-processor", plugin_type="processor"), + ], + ), + ], + } + ) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + entries = service.get_package_entries("calculator", "local", include_incompatible=True) + + assert [entry.name for entry in entries] == ["arithmetic-column", "arithmetic-processor"] + assert {entry.package.name for entry in entries} == {"data-designer-calculator"} + + +def test_get_package_entries_prefers_exact_package_name_over_package_alias() -> None: + repository = Mock(spec=PluginCatalogRepository) + repository.load_catalog.return_value = PluginCatalog.model_validate( + { + "schema_version": 2, + "packages": [ + _package( + package_name="calculator", + data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="plain-calculator", plugin_type="processor")], + ), + _package( + package_name="data-designer-calculator", + data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="namespaced-calculator", plugin_type="processor")], + ), + ], + } + ) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + entries = service.get_package_entries("calculator", "local", include_incompatible=True) + + assert [entry.name for entry in entries] == ["plain-calculator"] + assert entries[0].package.name == "calculator" + + +def test_get_package_entries_does_not_resolve_runtime_plugin_name_that_is_not_package_alias() -> None: + repository = Mock(spec=PluginCatalogRepository) + repository.load_catalog.return_value = PluginCatalog.model_validate( + { + "schema_version": 2, + "packages": [ + _package( + package_name="data-designer-calculator", + data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="arithmetic", plugin_type="processor")], + ), + ], + } + ) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + assert service.get_package_entries("arithmetic", "local", include_incompatible=True) == [] + + def test_group_entries_by_package_groups_multi_plugin_packages(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index e99c86c52..72342cff2 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -102,6 +102,49 @@ def test_build_auto_install_plan_chooses_pip_when_uv_is_unavailable(mock_which: mock_which.assert_called_once_with("uv") +def test_build_pip_uninstall_plan_uses_package_name_not_install_requirement() -> None: + requirement = ( + "data-designer-template @ " + "git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git@data-designer-template/v0.1.0" + ) + entry = _entry(package_name="data-designer-template", install={"requirement": requirement}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_uninstall_plan(entry, catalog, manager="pip") + + assert plan.command == [ + sys.executable, + "-m", + "pip", + "uninstall", + "--yes", + "data-designer-template", + ] + assert plan.package_name == "data-designer-template" + assert plan.manager == "pip" + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_uninstall_plan_chooses_uv_when_available(mock_which: Mock) -> None: + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_uninstall_plan(entry, catalog, manager="auto") + + assert plan.command == [ + "uv", + "pip", + "uninstall", + "--python", + sys.executable, + "data-designer-template", + ] + assert plan.manager == "uv" + mock_which.assert_called_once_with("uv") + + @patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") def test_build_uv_install_plan_targets_current_python_and_adds_catalog_index(mock_which: Mock) -> None: entry = _entry( @@ -154,6 +197,16 @@ def test_install_raises_when_runner_fails() -> None: service.install(plan) +def test_uninstall_raises_when_runner_fails() -> None: + service = PluginInstallService(runner=lambda command: 2) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + plan = service.build_uninstall_plan(entry, catalog, manager="pip") + + with pytest.raises(RuntimeError, match="status 2"): + service.uninstall(plan) + + @patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") @patch("data_designer.cli.services.plugin_install_service.importlib.invalidate_caches") def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( @@ -283,6 +336,46 @@ def test_verify_entry_points_verifies_multi_runtime_package_entries(mock_entry_p assert service.verify_entry_points(entries) is True +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") +@patch("data_designer.cli.services.plugin_install_service.importlib.invalidate_caches") +def test_verify_entry_points_removed_succeeds_when_declared_entries_are_absent( + mock_invalidate_caches: Mock, + mock_entry_points: Mock, +) -> None: + entry = _entry( + package_name="data-designer-template", + plugin_name="text-transform", + entry_point_name="text-transform", + entry_point_value="data_designer_template.plugin:plugin", + install={"requirement": "data-designer-template"}, + ) + mock_entry_points.return_value = [ + SimpleNamespace(name="other-plugin", value="other_package.plugin:plugin"), + ] + service = PluginInstallService() + + assert service.verify_entry_points_removed([entry]) is True + mock_invalidate_caches.assert_called_once_with() + mock_entry_points.assert_called_once_with(group="data_designer.plugins") + + +@patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") +def test_verify_entry_points_removed_fails_when_declared_entry_still_exists(mock_entry_points: Mock) -> None: + entry = _entry( + package_name="data-designer-template", + plugin_name="text-transform", + entry_point_name="text-transform", + entry_point_value="data_designer_template.plugin:plugin", + install={"requirement": "data-designer-template"}, + ) + mock_entry_points.return_value = [ + SimpleNamespace(name="text-transform", value="data_designer_template.plugin:plugin"), + ] + service = PluginInstallService() + + assert service.verify_entry_points_removed([entry]) is False + + def _entry( *, package_name: str, diff --git a/packages/data-designer/tests/cli/test_main.py b/packages/data-designer/tests/cli/test_main.py index c40178454..ec611631f 100644 --- a/packages/data-designer/tests/cli/test_main.py +++ b/packages/data-designer/tests/cli/test_main.py @@ -221,4 +221,5 @@ def test_app_help_keeps_config_and_plugins_commands_reachable() -> None: assert plugins_result.exit_code == 0 assert "list" in plugins_result.output assert "install" in plugins_result.output + assert "uninstall" in plugins_result.output assert "catalogs" in plugins_result.output From e2735ab18f8320007f66c939ad74deec474b5247 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 20:49:50 +0000 Subject: [PATCH 16/34] document plugin package aliases --- architecture/cli.md | 23 ++++++++++------ .../src/data_designer/cli/README.md | 27 +++++++++++++------ 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/architecture/cli.md b/architecture/cli.md index 3660c9e37..da7457f82 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -39,12 +39,12 @@ Plugin catalog commands use the same layering shape: | Layer | Role | Example | |-------|------|---------| -| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugins list/search/info/install/installed/catalogs` → `PluginCatalogController(DATA_DESIGNER_HOME)` | -| **Controller** | UX flow: catalog tables, package metadata, compatibility display, install confirmation | `PluginCatalogController` composes catalog + install services | -| **Service** | Domain rules: package-first flattening, compatibility checks, install planning, entry point verification | `PluginCatalogService`, `PluginInstallService` | +| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugins list/search/info/install/uninstall/installed/catalogs` → `PluginCatalogController(DATA_DESIGNER_HOME)` | +| **Controller** | UX flow: catalog tables, package metadata, compatibility display, package mutation confirmations | `PluginCatalogController` composes catalog + install services | +| **Service** | Domain rules: package-first flattening, compatibility checks, install/uninstall planning, entry point verification | `PluginCatalogService`, `PluginInstallService` | | **Repository** | File/cache I/O for catalog aliases and catalog documents | `PluginCatalogRepository` | -The built-in `nvidia` catalog points at `https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`. `NVIDIA-NeMo/DataDesignerPlugins` defines the package-first catalog shape: top-level packages carry install metadata, compatibility constraints, docs, and nested runtime plugins. The CLI flattens nested plugins for list/search display, but `info` and `install` resolve back to the package so installation targets the package requirement. +The built-in `nvidia` catalog points at `https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`. `NVIDIA-NeMo/DataDesignerPlugins` defines the package-first catalog shape: top-level packages carry install metadata, compatibility constraints, docs, and nested runtime plugins. The CLI flattens nested plugins for list/search display, but `info`, `install`, and `uninstall` resolve package names or package aliases so environment mutations target the package distribution. Package aliases come from the `data-designer-{alias}` package-name pattern; for example, `data-designer-calculator` can be addressed as `calculator`. ### Generation Commands @@ -82,13 +82,20 @@ User invokes command (e.g., `data-designer plugins list`) → PluginCatalogRepository reads local config and cached/remote catalog JSON ``` -### Plugin Install +### Plugin Install/Uninstall ``` -User invokes command (e.g., `data-designer plugins install text-transform`) - → PluginCatalogController resolves runtime plugin or package name +User invokes command (e.g., `data-designer plugins install calculator`) + → PluginCatalogController resolves the plugin package name or package alias → PluginCatalogService evaluates Python and Data Designer compatibility → PluginInstallService builds a pip/uv install plan for the package requirement - → PluginInstallService verifies declared entry points after installation + → PluginInstallService verifies declared package entry points after installation +``` + +``` +User invokes command (e.g., `data-designer plugins uninstall calculator`) + → PluginCatalogController resolves the plugin package name or package alias + → PluginInstallService builds a pip/uv uninstall plan for the package distribution + → PluginInstallService verifies declared package entry points are no longer discovered ``` ### Generation diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 179828a62..abaa6c5bc 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -54,7 +54,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `list.py`: List current configurations - `models.py`: Configure models - `providers.py`: Configure providers - - `plugins.py`: Discover and install plugins from catalogs + - `plugins.py`: Discover, install, and uninstall plugin packages from catalogs - `reset.py`: Reset/delete configurations #### 2. **Controllers** (`controllers/`) @@ -67,7 +67,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - **Files**: - `model_controller.py`: Orchestrates model configuration workflows - `provider_controller.py`: Orchestrates provider configuration workflows - - `plugin_catalog_controller.py`: Orchestrates plugin catalog browsing, alias management, and install workflows + - `plugin_catalog_controller.py`: Orchestrates plugin catalog browsing, alias management, and package workflows **Key Features**: - **Associated Resource Management**: When deleting a provider, the controller checks for associated models and prompts the user to delete them together @@ -84,7 +84,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `model_service.py`: Model configuration business logic - `provider_service.py`: Provider business logic - `plugin_catalog_service.py`: Plugin catalog discovery, search, compatibility checks, and installed entry point listing - - `plugin_install_service.py`: Plugin install-plan resolution, package-manager execution, and runtime verification + - `plugin_install_service.py`: Plugin install/uninstall plan resolution, package-manager execution, and runtime verification **Key Methods**: - `list_all()`: Get all configured items @@ -233,6 +233,11 @@ catalogs: Stores fetched plugin catalog payloads as JSON cache files keyed by catalog alias and URL hash. This prevents a re-pointed alias from serving stale catalog data from a previous URL. +Plugin package arguments accept either the full package name or the package +alias. For packages named `data-designer-{alias}`, the alias is `{alias}`. For +example, `data-designer-github` can be addressed as `github` in `info`, +`install`, and `uninstall`. + ## Usage Examples ### Configure Providers @@ -276,7 +281,7 @@ data-designer config list data-designer config reset ``` -### Discover and Install Plugins +### Discover, Install, and Uninstall Plugin Packages ```bash # List compatible plugin packages from the default NVIDIA catalog @@ -286,13 +291,19 @@ data-designer plugins list data-designer plugins --catalog research search transform # Show metadata, compatibility, docs, and exact install command -data-designer plugins info text-transform +data-designer plugins info github -# Install a plugin package from a catalog and verify declared runtime entry point discovery -data-designer plugins install text-transform --yes +# Install a plugin package from a catalog and verify package registration +data-designer plugins install github --yes # Preview the install command without mutating the environment -data-designer plugins install text-transform --dry-run +data-designer plugins install github --dry-run + +# Uninstall a plugin package from a catalog and verify package registration is removed +data-designer plugins uninstall github --yes + +# Preview the uninstall command without mutating the environment +data-designer plugins uninstall github --dry-run # Add and manage catalog aliases data-designer plugins catalogs add research https://github.com/acme/dd-plugins From b86a7a85ec391e7b430988612a95124de05e2ca5 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 20:53:30 +0000 Subject: [PATCH 17/34] address plugin catalog review feedback --- .../controllers/plugin_catalog_controller.py | 5 ++ .../src/data_designer/cli/plugin_catalog.py | 14 ++++- .../repositories/plugin_catalog_repository.py | 2 +- .../cli/services/plugin_install_service.py | 13 +++-- .../test_plugin_catalog_controller.py | 52 ++++++++++++++++++- .../test_plugin_catalog_repository.py | 30 +++++++++++ .../services/test_plugin_install_service.py | 5 +- 7 files changed, 114 insertions(+), 7 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index d3b0f35e4..6c5d80db4 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -130,6 +130,8 @@ def run_info( if entry.install.index_url is not None: console.print(f" Index URL: [bold]{entry.install.index_url}[/bold]") console.print(f" Install command: [bold]{shlex.join(plan.command)}[/bold]") + if plan.source_warning is not None: + print_warning(plan.source_warning) except ValueError as e: print_warning(str(e)) @@ -195,6 +197,9 @@ def run_install( console.print(f" Command: [bold]{shlex.join(plan.command)}[/bold]") self._display_compatibility(compatibility) + if plan.source_warning is not None: + print_warning(plan.source_warning) + if not catalog.trusted: print_warning( "This catalog is not marked trusted. Plugin package installation executes Python package code from " diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index ab46a97fc..b0cc9d2c3 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from urllib.parse import urlparse +from packaging.markers import InvalidMarker, Marker from packaging.requirements import InvalidRequirement, Requirement from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import InvalidName, canonicalize_name @@ -212,6 +213,7 @@ class InstallPlan: manager: str catalog_alias: str trusted_catalog: bool + source_warning: str | None = None @dataclass(frozen=True) @@ -475,7 +477,7 @@ def _catalog_data_designer_compatibility( f"expected {str(requirement.specifier)!r} from requirement" ) - marker = _required_catalog_nullable_string(f"{context}.marker", compatibility["marker"]) + marker = _catalog_marker(package_name, f"{context}.marker", compatibility["marker"]) expected_marker = str(requirement.marker) if requirement.marker is not None else None if marker != expected_marker: raise PluginCatalogError( @@ -483,6 +485,16 @@ def _catalog_data_designer_compatibility( ) +def _catalog_marker(package_name: str, context: str, value: object) -> str | None: + raw_marker = _required_catalog_nullable_string(context, value) + if raw_marker is None: + return None + try: + return str(Marker(raw_marker)) + except InvalidMarker as e: + raise PluginCatalogError(f"package {package_name!r} has invalid {context} {raw_marker!r}: {e}") from e + + def _catalog_http_url(context: str, value: object) -> str: url = _required_catalog_string(context, value) parsed = urlparse(url) diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py index a3ec4d051..4134b315b 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py @@ -141,7 +141,6 @@ def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> Pl try: payload = self._fetch_catalog_payload(catalog_config.url) - catalog = self._validate_catalog(payload, source=catalog_config.url) except (PluginCatalogError, OSError, ValueError): if not refresh: cached_catalog = self._load_cached_catalog(catalog_config, require_fresh=False) @@ -149,6 +148,7 @@ def load_catalog(self, alias: str | None = None, *, refresh: bool = False) -> Pl return cached_catalog raise + catalog = self._validate_catalog(payload, source=catalog_config.url) self._save_catalog_cache(catalog_config, payload) return catalog diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 8cce6bbf6..1f5d5ca75 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -22,6 +22,10 @@ ) InstallRunner = Callable[[list[str]], int] +PIP_EXTRA_INDEX_SOURCE_WARNING = ( + "pip --extra-index-url is not source-pinned; pip may choose a same-named package from another configured index. " + "Use uv or a direct reference when strict source selection is required." +) class PluginInstallService: @@ -39,7 +43,7 @@ def build_install_plan( ) -> InstallPlan: """Build the exact package-manager command for one catalog entry.""" resolved_manager = _resolve_manager(manager) - install_args, source_description = _install_args_for_entry(entry, resolved_manager) + install_args, source_description, source_warning = _install_args_for_entry(entry, resolved_manager) command = _base_command(resolved_manager) + install_args return InstallPlan( package_name=entry.package.name, @@ -48,6 +52,7 @@ def build_install_plan( manager=resolved_manager, catalog_alias=catalog.alias, trusted_catalog=catalog.trusted, + source_warning=source_warning, ) def build_uninstall_plan( @@ -178,18 +183,20 @@ def _base_uninstall_command(manager: str) -> list[str]: return [sys.executable, "-m", "pip", "uninstall", "--yes"] -def _install_args_for_entry(entry: PluginCatalogEntry, manager: str) -> tuple[list[str], str]: +def _install_args_for_entry(entry: PluginCatalogEntry, manager: str) -> tuple[list[str], str, str | None]: requirement = entry.install.requirement index_url = entry.install.index_url if index_url is None: - return [requirement], requirement + return [requirement], requirement, None if manager == "uv": return ( ["--default-index", PYPI_SIMPLE_INDEX_URL, "--index", index_url, requirement], f"{requirement} via {index_url}", + None, ) return ( ["--extra-index-url", index_url, requirement], f"{requirement} via {index_url}", + PIP_EXTRA_INDEX_SOURCE_WARNING, ) diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index d129eff5e..4a046dc66 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -179,6 +179,32 @@ def test_run_info_renders_package_metadata_with_nested_runtime_plugins( assert mock_console.print.call_count >= 1 +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.display_config_preview") +def test_run_info_warns_when_install_plan_has_source_warning( + mock_display_config_preview: MagicMock, + mock_console: MagicMock, + mock_print_warning: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [entry] + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = _plan( + catalog, + source_warning="pip source warning", + ) + + controller.run_info("text-transform", catalog_alias="local") + + mock_print_warning.assert_called_once_with("pip source warning") + mock_display_config_preview.assert_called_once() + assert mock_console.print.call_count >= 1 + + @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") def test_run_info_rejects_runtime_plugin_name_that_is_not_package_alias( mock_print_error: MagicMock, @@ -356,6 +382,29 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( assert mock_console.print.call_count >= 1 +@patch("data_designer.cli.controllers.plugin_catalog_controller.console") +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_install_warns_when_install_plan_has_source_warning( + mock_print_warning: MagicMock, + mock_console: MagicMock, + controller: PluginCatalogController, +) -> None: + entry = _entry() + catalog = _catalog(trusted=True) + controller.catalog_service.get_catalog.return_value = catalog + controller.catalog_service.get_package_entries.return_value = [entry] + controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) + controller.install_service.build_install_plan.return_value = _plan( + catalog, + source_warning="pip source warning", + ) + + controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) + + mock_print_warning.assert_called_once_with("pip source warning") + assert mock_console.print.call_count >= 1 + + @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") def test_run_install_warns_for_untrusted_catalog( @@ -574,7 +623,7 @@ def _catalog(*, trusted: bool) -> PluginCatalogConfig: ) -def _plan(catalog: PluginCatalogConfig) -> InstallPlan: +def _plan(catalog: PluginCatalogConfig, *, source_warning: str | None = None) -> InstallPlan: return InstallPlan( package_name="data-designer-text-transform", source_description="data-designer-text-transform", @@ -582,6 +631,7 @@ def _plan(catalog: PluginCatalogConfig) -> InstallPlan: manager="pip", catalog_alias=catalog.alias, trusted_catalog=catalog.trusted, + source_warning=source_warning, ) diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py index a0c8c045f..d02c222d5 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py @@ -135,6 +135,18 @@ def test_load_catalog_falls_back_to_stale_cache_when_refresh_fetch_fails(tmp_pat assert cached_catalog.plugins[0].name == "cached-transform" +def test_load_catalog_does_not_fall_back_to_stale_cache_when_fresh_catalog_is_invalid(tmp_path: Path) -> None: + catalog_path = _write_catalog(tmp_path, plugin_name="cached-transform") + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path), cache_ttl_seconds=0) + + repository.load_catalog("local") + catalog_path.write_text(json.dumps(_catalog_payload(schema_version=999, plugin_name="invalid-transform"))) + + with pytest.raises(PluginCatalogError, match="unsupported catalog schema_version"): + repository.load_catalog("local") + + def test_load_catalog_with_zero_cache_ttl_refreshes_source(tmp_path: Path) -> None: catalog_path = _write_catalog(tmp_path, plugin_name="text-transform") repository = PluginCatalogRepository(tmp_path) @@ -263,6 +275,24 @@ def test_load_catalog_accepts_schema_v2_package_catalog(tmp_path: Path) -> None: assert catalog.plugins[0].install.index_url == "https://docs.example.test/simple/" +def test_load_catalog_accepts_equivalent_data_designer_marker_quoting(tmp_path: Path) -> None: + package = _package_entry() + package["compatibility"]["data_designer"] = { + "requirement": "data-designer>=0.5.7; python_version < '3.12'", + "specifier": ">=0.5.7", + "marker": "python_version < '3.12'", + } + catalog_path = _write_catalog(tmp_path, packages=[package]) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + + catalog = repository.load_catalog("local", refresh=True) + + assert catalog.plugins[0].compatibility is not None + assert catalog.plugins[0].compatibility.data_designer is not None + assert catalog.plugins[0].compatibility.data_designer.marker == "python_version < '3.12'" + + def test_load_catalog_rejects_invalid_schema_v2_install_metadata(tmp_path: Path) -> None: catalog_path = _write_catalog( tmp_path, diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index 72342cff2..940c129e9 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -10,7 +10,7 @@ import pytest from data_designer.cli.plugin_catalog import PluginCatalogConfig, PluginCatalogEntry -from data_designer.cli.services.plugin_install_service import PluginInstallService +from data_designer.cli.services.plugin_install_service import PIP_EXTRA_INDEX_SOURCE_WARNING, PluginInstallService def test_build_pip_install_plan_uses_requirement_and_extra_index() -> None: @@ -40,6 +40,7 @@ def test_build_pip_install_plan_uses_requirement_and_extra_index() -> None: assert plan.source_description == ( "data-designer-template via https://nvidia-nemo.github.io/DataDesignerPlugins/simple/" ) + assert plan.source_warning == PIP_EXTRA_INDEX_SOURCE_WARNING def test_build_direct_reference_install_plan_uses_requirement_verbatim() -> None: @@ -55,6 +56,7 @@ def test_build_direct_reference_install_plan_uses_requirement_verbatim() -> None assert plan.command[-1] == requirement assert "--extra-index-url" not in plan.command + assert plan.source_warning is None @patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") @@ -86,6 +88,7 @@ def test_build_auto_install_plan_chooses_uv_when_available(mock_which: Mock) -> "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", "data-designer-template", ] + assert plan.source_warning is None mock_which.assert_called_once_with("uv") From a59d6aaad92e8aa2327aec09e11e77bad407705e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 20:55:21 +0000 Subject: [PATCH 18/34] prefer runtime plugin lookup matches --- .../cli/services/plugin_catalog_service.py | 6 ++--- .../services/test_plugin_catalog_service.py | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index 0c0fb5ed2..088326859 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -88,9 +88,9 @@ def get_entry( """Return a catalog entry by runtime plugin name or package name.""" entries = self.list_entries(catalog_alias, refresh=refresh, include_incompatible=True) canonical_name = canonicalize_name(name) - matches = [ - entry for entry in entries if entry.name == name or canonicalize_name(entry.package.name) == canonical_name - ] + runtime_matches = [entry for entry in entries if entry.name == name] + package_matches = [entry for entry in entries if canonicalize_name(entry.package.name) == canonical_name] + matches = runtime_matches or package_matches compatible_matches = [entry for entry in matches if self.evaluate_compatibility(entry).is_compatible] if compatible_matches: return sorted(compatible_matches, key=lambda entry: (canonicalize_name(entry.package.name), entry.name))[0] diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index df93051c9..7b8b11e61 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -181,6 +181,33 @@ def test_get_entry_resolves_package_name() -> None: assert entry.package.name == "data-designer-package-target" +def test_get_entry_prefers_runtime_plugin_name_over_package_name() -> None: + repository = Mock(spec=PluginCatalogRepository) + repository.load_catalog.return_value = PluginCatalog.model_validate( + { + "schema_version": 2, + "packages": [ + _package( + package_name="alpha", + data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="package-plugin", plugin_type="processor")], + ), + _package( + package_name="data-designer-runtime-source", + data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="alpha", plugin_type="processor")], + ), + ], + } + ) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + entry = service.get_entry("alpha", "local", include_incompatible=True) + + assert entry.name == "alpha" + assert entry.package.name == "data-designer-runtime-source" + + def test_get_package_entries_resolves_package_alias() -> None: repository = Mock(spec=PluginCatalogRepository) repository.load_catalog.return_value = PluginCatalog.model_validate( From 4c5da447b61c6921d11e0c0ecabb785053f4814a Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 21:08:13 +0000 Subject: [PATCH 19/34] rename plugins command to plugin --- architecture/cli.md | 10 ++-- .../src/data_designer/cli/README.md | 26 ++++----- .../cli/commands/{plugins.py => plugin.py} | 2 +- .../src/data_designer/cli/main.py | 32 +++++----- ...gins_command.py => test_plugin_command.py} | 58 +++++++++---------- packages/data-designer/tests/cli/test_main.py | 39 ++++++++----- 6 files changed, 87 insertions(+), 80 deletions(-) rename packages/data-designer/src/data_designer/cli/commands/{plugins.py => plugin.py} (99%) rename packages/data-designer/tests/cli/commands/{test_plugins_command.py => test_plugin_command.py} (63%) diff --git a/architecture/cli.md b/architecture/cli.md index da7457f82..f5a3a26a0 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -1,6 +1,6 @@ # CLI -The CLI (`data-designer`) provides an interactive command-line interface for configuring models, providers, tools, and personas, discovering/installing plugins from catalogs, and running dataset generation. It uses a layered architecture for setup workflows and delegates generation to the public `DataDesigner` API. +The CLI (`data-designer`) provides an interactive command-line interface for configuring models, providers, tools, and personas, discovering/installing plugin packages from catalogs, and running dataset generation. It uses a layered architecture for setup workflows and delegates generation to the public `DataDesigner` API. Source: `packages/data-designer/src/data_designer/cli/` @@ -39,7 +39,7 @@ Plugin catalog commands use the same layering shape: | Layer | Role | Example | |-------|------|---------| -| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugins list/search/info/install/uninstall/installed/catalogs` → `PluginCatalogController(DATA_DESIGNER_HOME)` | +| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugin list/search/info/install/uninstall/installed/catalogs` → `PluginCatalogController(DATA_DESIGNER_HOME)` | | **Controller** | UX flow: catalog tables, package metadata, compatibility display, package mutation confirmations | `PluginCatalogController` composes catalog + install services | | **Service** | Domain rules: package-first flattening, compatibility checks, install/uninstall planning, entry point verification | `PluginCatalogService`, `PluginInstallService` | | **Repository** | File/cache I/O for catalog aliases and catalog documents | `PluginCatalogRepository` | @@ -75,7 +75,7 @@ User invokes command (e.g., `data-designer config models`) ### Plugin Catalog Discovery ``` -User invokes command (e.g., `data-designer plugins list`) +User invokes command (e.g., `data-designer plugin list`) → Command function wires DATA_DESIGNER_HOME and catalog options → PluginCatalogController resolves the catalog alias → PluginCatalogService loads and filters package-first catalog entries @@ -84,7 +84,7 @@ User invokes command (e.g., `data-designer plugins list`) ### Plugin Install/Uninstall ``` -User invokes command (e.g., `data-designer plugins install calculator`) +User invokes command (e.g., `data-designer plugin install calculator`) → PluginCatalogController resolves the plugin package name or package alias → PluginCatalogService evaluates Python and Data Designer compatibility → PluginInstallService builds a pip/uv install plan for the package requirement @@ -92,7 +92,7 @@ User invokes command (e.g., `data-designer plugins install calculator`) ``` ``` -User invokes command (e.g., `data-designer plugins uninstall calculator`) +User invokes command (e.g., `data-designer plugin uninstall calculator`) → PluginCatalogController resolves the plugin package name or package alias → PluginInstallService builds a pip/uv uninstall plan for the package distribution → PluginInstallService verifies declared package entry points are no longer discovered diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index abaa6c5bc..59e977587 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -19,7 +19,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis ``` ┌─────────────────────────────────────────────────────────────┐ │ Commands │ -│ Entry points for CLI commands (list, providers, plugins) │ +│ Entry points for CLI commands (list, providers, plugin) │ └─────────────────────────────────────────────────────────────┘ │ ▼ @@ -54,7 +54,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `list.py`: List current configurations - `models.py`: Configure models - `providers.py`: Configure providers - - `plugins.py`: Discover, install, and uninstall plugin packages from catalogs + - `plugin.py`: Discover, install, and uninstall plugin packages from catalogs - `reset.py`: Reset/delete configurations #### 2. **Controllers** (`controllers/`) @@ -285,31 +285,31 @@ data-designer config reset ```bash # List compatible plugin packages from the default NVIDIA catalog -data-designer plugins list +data-designer plugin list # Search a specific catalog -data-designer plugins --catalog research search transform +data-designer plugin --catalog research search transform # Show metadata, compatibility, docs, and exact install command -data-designer plugins info github +data-designer plugin info github # Install a plugin package from a catalog and verify package registration -data-designer plugins install github --yes +data-designer plugin install github --yes # Preview the install command without mutating the environment -data-designer plugins install github --dry-run +data-designer plugin install github --dry-run # Uninstall a plugin package from a catalog and verify package registration is removed -data-designer plugins uninstall github --yes +data-designer plugin uninstall github --yes # Preview the uninstall command without mutating the environment -data-designer plugins uninstall github --dry-run +data-designer plugin uninstall github --dry-run # Add and manage catalog aliases -data-designer plugins catalogs add research https://github.com/acme/dd-plugins -data-designer plugins catalogs list -data-designer plugins catalogs remove research +data-designer plugin catalogs add research https://github.com/acme/dd-plugins +data-designer plugin catalogs list +data-designer plugin catalogs remove research # List installed runtime plugin entry points without importing plugin modules -data-designer plugins installed +data-designer plugin installed ``` diff --git a/packages/data-designer/src/data_designer/cli/commands/plugins.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py similarity index 99% rename from packages/data-designer/src/data_designer/cli/commands/plugins.py rename to packages/data-designer/src/data_designer/cli/commands/plugin.py index 6f3bc9405..cb964f1ef 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugins.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -254,7 +254,7 @@ def _resolve_catalog_alias(ctx: typer.Context, catalog_alias: str | None) -> str def _parent_catalog_alias(ctx: typer.Context) -> str | None: - """Return --catalog from the plugins parent command when present.""" + """Return --catalog from the plugin parent command when present.""" parent = ctx.parent while parent is not None: diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index 77adc493d..22473b910 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -141,39 +141,39 @@ def _is_version_request(args: list[str]) -> bool: no_args_is_help=True, ) -# Create plugins command group -plugins_app = typer.Typer( - name="plugins", +# Create plugin command group +plugin_app = typer.Typer( + name="plugin", help="Discover, install, and uninstall Data Designer plugin packages from catalogs", cls=create_lazy_typer_group( { "list": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "list_command", "help": "List plugin packages from a catalog", }, "search": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "search_command", "help": "Search plugin packages from a catalog", }, "info": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "info_command", "help": "Show plugin package metadata and install plan", }, "install": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "install_command", "help": "Install a plugin package and verify package registration", }, "uninstall": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "uninstall_command", "help": "Uninstall a plugin package and verify package registration is removed", }, "installed": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "installed_command", "help": "List installed runtime plugin entry points", }, @@ -183,8 +183,8 @@ def _is_version_request(args: list[str]) -> bool: ) -@plugins_app.callback() -def plugins_callback( +@plugin_app.callback() +def plugin_callback( catalog: str | None = typer.Option( None, "--catalog", @@ -200,17 +200,17 @@ def plugins_callback( cls=create_lazy_typer_group( { "list": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "catalogs_list_command", "help": "List configured plugin catalogs", }, "add": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "catalogs_add_command", "help": "Add a plugin catalog alias", }, "remove": { - "module": f"{_CMD}.plugins", + "module": f"{_CMD}.plugin", "attr": "catalogs_remove_command", "help": "Remove a plugin catalog alias", }, @@ -245,12 +245,12 @@ def _build_agent_lazy_group(prefix: str) -> dict[str, dict[str, str]]: ) agent_app.add_typer(agent_state_app, name="state") -plugins_app.add_typer(plugin_catalogs_app, name="catalogs") +plugin_app.add_typer(plugin_catalogs_app, name="catalogs") # Add setup command groups app.add_typer(config_app, name="config", rich_help_panel="Setup") app.add_typer(download_app, name="download", rich_help_panel="Setup") -app.add_typer(plugins_app, name="plugins", rich_help_panel="Setup") +app.add_typer(plugin_app, name="plugin", rich_help_panel="Setup") app.add_typer(agent_app, name="agent", rich_help_panel="Agent") diff --git a/packages/data-designer/tests/cli/commands/test_plugins_command.py b/packages/data-designer/tests/cli/commands/test_plugin_command.py similarity index 63% rename from packages/data-designer/tests/cli/commands/test_plugins_command.py rename to packages/data-designer/tests/cli/commands/test_plugin_command.py index 2fffd17f2..202043774 100644 --- a/packages/data-designer/tests/cli/commands/test_plugins_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugin_command.py @@ -12,12 +12,12 @@ runner = CliRunner() -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_list_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_list_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "--catalog", "research", "list", "--refresh", "--include-incompatible"]) + result = runner.invoke(app, ["plugin", "--catalog", "research", "list", "--refresh", "--include-incompatible"]) assert result.exit_code == 0 mock_ctrl.run_list.assert_called_once_with( @@ -27,12 +27,12 @@ def test_plugins_list_command_delegates_to_controller(mock_ctrl_cls: MagicMock) ) -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_search_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_search_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "search", "github", "--catalog", "research"]) + result = runner.invoke(app, ["plugin", "search", "github", "--catalog", "research"]) assert result.exit_code == 0 mock_ctrl.run_search.assert_called_once_with( @@ -43,14 +43,14 @@ def test_plugins_search_command_delegates_to_controller(mock_ctrl_cls: MagicMock ) -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_install_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl result = runner.invoke( app, - ["plugins", "install", "data-designer-text-transform", "--manager", "pip", "--yes", "--dry-run"], + ["plugin", "install", "data-designer-text-transform", "--manager", "pip", "--yes", "--dry-run"], ) assert result.exit_code == 0 @@ -65,14 +65,14 @@ def test_plugins_install_command_delegates_to_controller(mock_ctrl_cls: MagicMoc ) -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_uninstall_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_uninstall_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl result = runner.invoke( app, - ["plugins", "uninstall", "data-designer-text-transform", "--manager", "pip", "--yes", "--dry-run"], + ["plugin", "uninstall", "data-designer-text-transform", "--manager", "pip", "--yes", "--dry-run"], ) assert result.exit_code == 0 @@ -86,8 +86,8 @@ def test_plugins_uninstall_command_delegates_to_controller(mock_ctrl_cls: MagicM ) -def test_plugins_info_help_uses_package_argument() -> None: - result = runner.invoke(app, ["plugins", "info", "--help"]) +def test_plugin_info_help_uses_package_argument() -> None: + result = runner.invoke(app, ["plugin", "info", "--help"]) assert result.exit_code == 0 assert "PACKAGE" in result.output @@ -95,8 +95,8 @@ def test_plugins_info_help_uses_package_argument() -> None: assert "runtime plugin name" not in result.output -def test_plugins_install_help_uses_package_first_wording() -> None: - result = runner.invoke(app, ["plugins", "install", "--help"]) +def test_plugin_install_help_uses_package_first_wording() -> None: + result = runner.invoke(app, ["plugin", "install", "--help"]) assert result.exit_code == 0 assert "PACKAGE" in result.output @@ -106,8 +106,8 @@ def test_plugins_install_help_uses_package_first_wording() -> None: assert "package when compatibility" in result.output -def test_plugins_uninstall_help_uses_package_first_wording() -> None: - result = runner.invoke(app, ["plugins", "uninstall", "--help"]) +def test_plugin_uninstall_help_uses_package_first_wording() -> None: + result = runner.invoke(app, ["plugin", "uninstall", "--help"]) assert result.exit_code == 0 assert "PACKAGE" in result.output @@ -116,15 +116,15 @@ def test_plugins_uninstall_help_uses_package_first_wording() -> None: assert "Print the uninstall plan" in result.output -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl result = runner.invoke( app, [ - "plugins", + "plugin", "catalogs", "add", "research", @@ -144,16 +144,16 @@ def test_plugins_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: Mag ) -@patch("data_designer.cli.commands.plugins.print_info") -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_installed_warns_when_parent_catalog_is_unused( +@patch("data_designer.cli.commands.plugin.print_info") +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_installed_warns_when_parent_catalog_is_unused( mock_ctrl_cls: MagicMock, mock_print_info: MagicMock, ) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "--catalog", "research", "installed"]) + result = runner.invoke(app, ["plugin", "--catalog", "research", "installed"]) assert result.exit_code == 0 mock_print_info.assert_called_once_with( @@ -162,16 +162,16 @@ def test_plugins_installed_warns_when_parent_catalog_is_unused( mock_ctrl.run_installed.assert_called_once_with() -@patch("data_designer.cli.commands.plugins.print_info") -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_plugins_catalogs_list_warns_when_parent_catalog_is_unused( +@patch("data_designer.cli.commands.plugin.print_info") +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_plugin_catalogs_list_warns_when_parent_catalog_is_unused( mock_ctrl_cls: MagicMock, mock_print_info: MagicMock, ) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugins", "--catalog", "research", "catalogs", "list"]) + result = runner.invoke(app, ["plugin", "--catalog", "research", "catalogs", "list"]) assert result.exit_code == 0 mock_print_info.assert_called_once_with( diff --git a/packages/data-designer/tests/cli/test_main.py b/packages/data-designer/tests/cli/test_main.py index ec611631f..ac49ef37b 100644 --- a/packages/data-designer/tests/cli/test_main.py +++ b/packages/data-designer/tests/cli/test_main.py @@ -32,9 +32,9 @@ def test_main_bootstraps_before_running_app(mock_bootstrap: Mock, mock_app: Mock @patch("data_designer.cli.main.app") @patch("data_designer.cli.main.ensure_cli_default_model_settings") -def test_main_bootstraps_for_plugins_commands(mock_bootstrap: Mock, mock_app: Mock) -> None: - """Plugin commands still run through CLI default setup before Typer dispatch.""" - with patch("sys.argv", ["data-designer", "plugins", "list"]): +def test_main_bootstraps_for_plugin_commands(mock_bootstrap: Mock, mock_app: Mock) -> None: + """The plugin command still runs through CLI default setup before Typer dispatch.""" + with patch("sys.argv", ["data-designer", "plugin", "list"]): main() mock_bootstrap.assert_called_once_with() @@ -180,15 +180,15 @@ def test_app_dispatches_lazy_create_command(mock_controller_cls: Mock) -> None: ) -@patch("data_designer.cli.commands.plugins.PluginCatalogController") -def test_app_dispatches_lazy_plugins_list_command(mock_controller_cls: Mock) -> None: - """The plugins group lazily resolves command callbacks without loading a catalog.""" +@patch("data_designer.cli.commands.plugin.PluginCatalogController") +def test_app_dispatches_lazy_plugin_list_command(mock_controller_cls: Mock) -> None: + """The plugin group lazily resolves command callbacks without loading a catalog.""" mock_controller = Mock() mock_controller_cls.return_value = mock_controller result = runner.invoke( app, - ["plugins", "--catalog", "local", "list", "--refresh", "--include-incompatible"], + ["plugin", "--catalog", "local", "list", "--refresh", "--include-incompatible"], ) assert result.exit_code == 0 @@ -199,27 +199,34 @@ def test_app_dispatches_lazy_plugins_list_command(mock_controller_cls: Mock) -> ) -@patch("data_designer.cli.commands.plugins.PluginCatalogController") +@patch("data_designer.cli.commands.plugin.PluginCatalogController") def test_app_dispatches_lazy_plugin_catalogs_list_command(mock_controller_cls: Mock) -> None: """Nested plugin catalog commands resolve through the lazy command group.""" mock_controller = Mock() mock_controller_cls.return_value = mock_controller - result = runner.invoke(app, ["plugins", "catalogs", "list"]) + result = runner.invoke(app, ["plugin", "catalogs", "list"]) assert result.exit_code == 0 mock_controller.run_catalogs_list.assert_called_once_with() -def test_app_help_keeps_config_and_plugins_commands_reachable() -> None: +def test_app_help_keeps_config_and_plugin_commands_reachable() -> None: config_result = runner.invoke(app, ["config", "--help"]) - plugins_result = runner.invoke(app, ["plugins", "--help"]) + plugin_result = runner.invoke(app, ["plugin", "--help"]) assert config_result.exit_code == 0 assert "providers" in config_result.output assert "models" in config_result.output - assert plugins_result.exit_code == 0 - assert "list" in plugins_result.output - assert "install" in plugins_result.output - assert "uninstall" in plugins_result.output - assert "catalogs" in plugins_result.output + assert plugin_result.exit_code == 0 + assert "list" in plugin_result.output + assert "install" in plugin_result.output + assert "uninstall" in plugin_result.output + assert "catalogs" in plugin_result.output + + +def test_app_does_not_expose_legacy_plugins_command() -> None: + result = runner.invoke(app, ["plugins", "--help"]) + + assert result.exit_code != 0 + assert "No such command" in result.output From b9e74b32c601a5d786ca3764fc070dcd477147d2 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 21:08:21 +0000 Subject: [PATCH 20/34] show plugin package descriptions --- .../data_designer/cli/controllers/plugin_catalog_controller.py | 2 ++ .../tests/cli/controllers/test_plugin_catalog_controller.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 6c5d80db4..741a567ef 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -463,6 +463,7 @@ def _display_empty_search_state( def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: table = Table(title="Catalog Plugin Packages", border_style=NordColor.NORD8.value) table.add_column("Package", style=NordColor.NORD14.value, no_wrap=True) + table.add_column("Description", style=NordColor.NORD4.value) table.add_column("Runtime Plugins", style=NordColor.NORD9.value) table.add_column("Compatible", style=NordColor.NORD13.value, no_wrap=True) table.add_column("Docs", style=NordColor.NORD7.value) @@ -473,6 +474,7 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: docs_url = entry.docs.url if entry.docs is not None and entry.docs.url is not None else "" table.add_row( entry.package.name, + entry.description, _format_runtime_plugins(package_entries), "yes" if compatibility.is_compatible else "no", docs_url, diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 4a046dc66..fa98e2c2c 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -109,10 +109,12 @@ def test_run_list_renders_package_first_catalog_table( assert printed_tables[0].title == "Catalog Plugin Packages" assert [column.header for column in printed_tables[0].columns] == [ "Package", + "Description", "Runtime Plugins", "Compatible", "Docs", ] + assert list(printed_tables[0].columns[1].cells) == ["Transform text records"] controller.catalog_service.group_entries_by_package.assert_called_once_with(package_entries) From 86a0776c0a6528c059020df76cecb33f160ce2a7 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 21:27:01 +0000 Subject: [PATCH 21/34] rename plugin catalogs command --- architecture/cli.md | 2 +- .../data-designer/src/data_designer/cli/README.md | 6 +++--- .../src/data_designer/cli/commands/plugin.py | 12 ++++++------ .../cli/controllers/plugin_catalog_controller.py | 6 +++--- .../data-designer/src/data_designer/cli/main.py | 12 ++++++------ .../tests/cli/commands/test_plugin_command.py | 12 ++++++------ .../controllers/test_plugin_catalog_controller.py | 8 ++++---- packages/data-designer/tests/cli/test_main.py | 15 +++++++++++---- 8 files changed, 40 insertions(+), 33 deletions(-) diff --git a/architecture/cli.md b/architecture/cli.md index f5a3a26a0..405c7d4bd 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -39,7 +39,7 @@ Plugin catalog commands use the same layering shape: | Layer | Role | Example | |-------|------|---------| -| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugin list/search/info/install/uninstall/installed/catalogs` → `PluginCatalogController(DATA_DESIGNER_HOME)` | +| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugin list/search/info/install/uninstall/installed/catalog` → `PluginCatalogController(DATA_DESIGNER_HOME)` | | **Controller** | UX flow: catalog tables, package metadata, compatibility display, package mutation confirmations | `PluginCatalogController` composes catalog + install services | | **Service** | Domain rules: package-first flattening, compatibility checks, install/uninstall planning, entry point verification | `PluginCatalogService`, `PluginInstallService` | | **Repository** | File/cache I/O for catalog aliases and catalog documents | `PluginCatalogRepository` | diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 59e977587..bc318da22 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -306,9 +306,9 @@ data-designer plugin uninstall github --yes data-designer plugin uninstall github --dry-run # Add and manage catalog aliases -data-designer plugin catalogs add research https://github.com/acme/dd-plugins -data-designer plugin catalogs list -data-designer plugin catalogs remove research +data-designer plugin catalog add research https://github.com/acme/dd-plugins +data-designer plugin catalog list +data-designer plugin catalog remove research # List installed runtime plugin entry points without importing plugin modules data-designer plugin installed diff --git a/packages/data-designer/src/data_designer/cli/commands/plugin.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py index cb964f1ef..a1ed6d8de 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugin.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -200,14 +200,14 @@ def installed_command(ctx: typer.Context) -> None: controller.run_installed() -def catalogs_list_command(ctx: typer.Context) -> None: +def catalog_list_command(ctx: typer.Context) -> None: """List configured plugin catalogs.""" _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) - controller.run_catalogs_list() + controller.run_catalog_list() -def catalogs_add_command( +def catalog_add_command( ctx: typer.Context, alias: str = typer.Argument(help="Local alias for the plugin catalog."), url: str = typer.Argument( @@ -228,7 +228,7 @@ def catalogs_add_command( """Add a plugin catalog alias.""" _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) - controller.run_catalogs_add( + controller.run_catalog_add( alias=alias, url=url, trusted=trusted, @@ -236,14 +236,14 @@ def catalogs_add_command( ) -def catalogs_remove_command( +def catalog_remove_command( ctx: typer.Context, alias: str = typer.Argument(help="Plugin catalog alias to remove."), ) -> None: """Remove a plugin catalog alias.""" _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") controller = PluginCatalogController(DATA_DESIGNER_HOME) - controller.run_catalogs_remove(alias=alias) + controller.run_catalog_remove(alias=alias) def _resolve_catalog_alias(ctx: typer.Context, catalog_alias: str | None) -> str | None: diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 741a567ef..80a531ec1 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -295,7 +295,7 @@ def run_installed(self) -> None: return self._display_installed_plugins(installed_plugins) - def run_catalogs_list(self) -> None: + def run_catalog_list(self) -> None: """List configured plugin catalogs.""" print_header("Data Designer Plugin Catalogs") try: @@ -319,7 +319,7 @@ def run_catalogs_list(self) -> None: ) console.print(table) - def run_catalogs_add( + def run_catalog_add( self, *, alias: str, @@ -348,7 +348,7 @@ def run_catalogs_add( print_success(f"Plugin catalog {catalog.alias!r} added") print_info(f"Catalog: {catalog.url}") - def run_catalogs_remove(self, *, alias: str) -> None: + def run_catalog_remove(self, *, alias: str) -> None: """Remove a plugin catalog alias.""" try: self.catalog_service.remove_catalog(alias) diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index 22473b910..0c46073f6 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -194,24 +194,24 @@ def plugin_callback( _ = catalog -plugin_catalogs_app = typer.Typer( - name="catalogs", +plugin_catalog_app = typer.Typer( + name="catalog", help="Manage plugin catalog aliases", cls=create_lazy_typer_group( { "list": { "module": f"{_CMD}.plugin", - "attr": "catalogs_list_command", + "attr": "catalog_list_command", "help": "List configured plugin catalogs", }, "add": { "module": f"{_CMD}.plugin", - "attr": "catalogs_add_command", + "attr": "catalog_add_command", "help": "Add a plugin catalog alias", }, "remove": { "module": f"{_CMD}.plugin", - "attr": "catalogs_remove_command", + "attr": "catalog_remove_command", "help": "Remove a plugin catalog alias", }, } @@ -245,7 +245,7 @@ def _build_agent_lazy_group(prefix: str) -> dict[str, dict[str, str]]: ) agent_app.add_typer(agent_state_app, name="state") -plugin_app.add_typer(plugin_catalogs_app, name="catalogs") +plugin_app.add_typer(plugin_catalog_app, name="catalog") # Add setup command groups app.add_typer(config_app, name="config", rich_help_panel="Setup") diff --git a/packages/data-designer/tests/cli/commands/test_plugin_command.py b/packages/data-designer/tests/cli/commands/test_plugin_command.py index 202043774..5b8cac2fa 100644 --- a/packages/data-designer/tests/cli/commands/test_plugin_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugin_command.py @@ -117,7 +117,7 @@ def test_plugin_uninstall_help_uses_package_first_wording() -> None: @patch("data_designer.cli.commands.plugin.PluginCatalogController") -def test_plugin_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: +def test_plugin_catalog_add_command_delegates_to_controller(mock_ctrl_cls: MagicMock) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl @@ -125,7 +125,7 @@ def test_plugin_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: Magi app, [ "plugin", - "catalogs", + "catalog", "add", "research", "https://github.com/acme/dd-plugins", @@ -136,7 +136,7 @@ def test_plugin_catalogs_add_command_delegates_to_controller(mock_ctrl_cls: Magi ) assert result.exit_code == 0 - mock_ctrl.run_catalogs_add.assert_called_once_with( + mock_ctrl.run_catalog_add.assert_called_once_with( alias="research", url="https://github.com/acme/dd-plugins", trusted=True, @@ -164,17 +164,17 @@ def test_plugin_installed_warns_when_parent_catalog_is_unused( @patch("data_designer.cli.commands.plugin.print_info") @patch("data_designer.cli.commands.plugin.PluginCatalogController") -def test_plugin_catalogs_list_warns_when_parent_catalog_is_unused( +def test_plugin_catalog_list_warns_when_parent_catalog_is_unused( mock_ctrl_cls: MagicMock, mock_print_info: MagicMock, ) -> None: mock_ctrl = MagicMock() mock_ctrl_cls.return_value = mock_ctrl - result = runner.invoke(app, ["plugin", "--catalog", "research", "catalogs", "list"]) + result = runner.invoke(app, ["plugin", "--catalog", "research", "catalog", "list"]) assert result.exit_code == 0 mock_print_info.assert_called_once_with( "Ignoring --catalog 'research'; catalog management commands operate on aliases directly." ) - mock_ctrl.run_catalogs_list.assert_called_once_with() + mock_ctrl.run_catalog_list.assert_called_once_with() diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index fa98e2c2c..730ad0b93 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -585,14 +585,14 @@ def test_run_uninstall_warns_when_entry_points_remain( @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") -def test_run_catalogs_add_wraps_invalid_alias_validation_error( +def test_run_catalog_add_wraps_invalid_alias_validation_error( mock_print_error: MagicMock, tmp_path: Path, ) -> None: plugin_controller = PluginCatalogController(tmp_path) with pytest.raises(typer.Exit) as exc_info: - plugin_controller.run_catalogs_add( + plugin_controller.run_catalog_add( alias="foo/bar", url="https://github.com/acme/dd-plugins", trusted=False, @@ -604,14 +604,14 @@ def test_run_catalogs_add_wraps_invalid_alias_validation_error( @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") -def test_run_catalogs_list_wraps_registry_load_error( +def test_run_catalog_list_wraps_registry_load_error( mock_print_error: MagicMock, controller: PluginCatalogController, ) -> None: controller.catalog_service.list_catalogs.side_effect = PluginCatalogError("bad registry") with pytest.raises(typer.Exit) as exc_info: - controller.run_catalogs_list() + controller.run_catalog_list() assert exc_info.value.exit_code == 1 mock_print_error.assert_called_once_with("Failed to list plugin catalogs: bad registry") diff --git a/packages/data-designer/tests/cli/test_main.py b/packages/data-designer/tests/cli/test_main.py index ac49ef37b..db805adb7 100644 --- a/packages/data-designer/tests/cli/test_main.py +++ b/packages/data-designer/tests/cli/test_main.py @@ -200,15 +200,15 @@ def test_app_dispatches_lazy_plugin_list_command(mock_controller_cls: Mock) -> N @patch("data_designer.cli.commands.plugin.PluginCatalogController") -def test_app_dispatches_lazy_plugin_catalogs_list_command(mock_controller_cls: Mock) -> None: +def test_app_dispatches_lazy_plugin_catalog_list_command(mock_controller_cls: Mock) -> None: """Nested plugin catalog commands resolve through the lazy command group.""" mock_controller = Mock() mock_controller_cls.return_value = mock_controller - result = runner.invoke(app, ["plugin", "catalogs", "list"]) + result = runner.invoke(app, ["plugin", "catalog", "list"]) assert result.exit_code == 0 - mock_controller.run_catalogs_list.assert_called_once_with() + mock_controller.run_catalog_list.assert_called_once_with() def test_app_help_keeps_config_and_plugin_commands_reachable() -> None: @@ -222,7 +222,7 @@ def test_app_help_keeps_config_and_plugin_commands_reachable() -> None: assert "list" in plugin_result.output assert "install" in plugin_result.output assert "uninstall" in plugin_result.output - assert "catalogs" in plugin_result.output + assert "catalog" in plugin_result.output def test_app_does_not_expose_legacy_plugins_command() -> None: @@ -230,3 +230,10 @@ def test_app_does_not_expose_legacy_plugins_command() -> None: assert result.exit_code != 0 assert "No such command" in result.output + + +def test_plugin_does_not_expose_legacy_catalogs_command() -> None: + result = runner.invoke(app, ["plugin", "catalogs", "--help"]) + + assert result.exit_code != 0 + assert "No such command" in result.output From 47f843691e5edcbe00f9eee01997df3781671c85 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 22:37:46 +0000 Subject: [PATCH 22/34] add protected plugin package installs --- .../src/data_designer/cli/commands/plugin.py | 10 +- .../controllers/plugin_catalog_controller.py | 43 ++- .../src/data_designer/cli/plugin_catalog.py | 17 + .../cli/services/plugin_install_service.py | 338 ++++++++++++++++-- .../tests/cli/commands/test_plugin_command.py | 4 +- .../test_plugin_catalog_controller.py | 27 +- .../services/test_plugin_install_service.py | 331 ++++++++++++++++- 7 files changed, 712 insertions(+), 58 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/commands/plugin.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py index a1ed6d8de..e574ea0dd 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugin.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -115,7 +115,7 @@ def install_command( "auto", "--manager", click_type=click.Choice(["auto", "uv", "pip"]), - help="Package manager to use for installation.", + help="Package manager to use. uv adds to the active project when one is detected; pip mutates the environment.", ), yes: bool = typer.Option( False, @@ -128,11 +128,6 @@ def install_command( "--dry-run", help="Print the install plan without mutating the current environment.", ), - force: bool = typer.Option( - False, - "--force", - help="Allow installing a catalog package when compatibility checks fail.", - ), ) -> None: """Install one Data Designer plugin package, then verify package registration.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) @@ -143,7 +138,6 @@ def install_command( manager=manager, yes=yes, dry_run=dry_run, - force=force, ) @@ -167,7 +161,7 @@ def uninstall_command( "auto", "--manager", click_type=click.Choice(["auto", "uv", "pip"]), - help="Package manager to use for uninstallation.", + help="Package manager to use. uv removes from the active project when one is detected; pip mutates the environment.", ), yes: bool = typer.Option( False, diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 80a531ec1..bf65c71a0 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -129,6 +129,9 @@ def run_info( console.print(f" Requirement: [bold]{entry.install.requirement}[/bold]") if entry.install.index_url is not None: console.print(f" Index URL: [bold]{entry.install.index_url}[/bold]") + console.print(f" Install target: [bold]{_target_description(plan.install_mode, plan.project_root)}[/bold]") + if plan.data_designer_protection is not None: + console.print(f" Data Designer: [bold]{plan.data_designer_protection}[/bold]") console.print(f" Install command: [bold]{shlex.join(plan.command)}[/bold]") if plan.source_warning is not None: print_warning(plan.source_warning) @@ -163,7 +166,6 @@ def run_install( manager: str = "auto", yes: bool = False, dry_run: bool = False, - force: bool = False, ) -> None: """Install one plugin package from the catalog.""" catalog = self._get_catalog_or_exit(catalog_alias) @@ -176,7 +178,7 @@ def run_install( entry = package_entries[0] compatibility = self.catalog_service.evaluate_compatibility(entry) - if not compatibility.is_compatible and not force and not dry_run: + if not compatibility.is_compatible and not dry_run: print_error(f"Plugin package {entry.package.name!r} is not compatible with this environment") for reason in compatibility.reasons: console.print(f" - {reason}") @@ -194,6 +196,9 @@ def run_install( console.print(f" Requirement: [bold]{entry.install.requirement}[/bold]") if entry.install.index_url is not None: console.print(f" Index URL: [bold]{entry.install.index_url}[/bold]") + console.print(f" Install target: [bold]{_target_description(plan.install_mode, plan.project_root)}[/bold]") + if plan.data_designer_protection is not None: + console.print(f" Data Designer: [bold]{plan.data_designer_protection}[/bold]") console.print(f" Command: [bold]{shlex.join(plan.command)}[/bold]") self._display_compatibility(compatibility) @@ -207,15 +212,19 @@ def run_install( ) if dry_run: - if not compatibility.is_compatible and not force: + if not compatibility.is_compatible: print_warning( - "Dry run complete; no changes made. A real install would be blocked unless you pass --force." + "Dry run complete; no changes made. A real install would be blocked because compatibility " + "checks failed." ) else: print_info("Dry run complete; no changes made") return - if not yes and not confirm_action("Install this package into the current Python environment?", default=False): + if not yes and not confirm_action( + f"Install this package into the {_target_description(plan.install_mode, plan.project_root)}?", + default=False, + ): print_info("No changes made") return @@ -262,13 +271,17 @@ def run_uninstall( print_header("Uninstall Data Designer Plugin Package") console.print(f" Package: [bold]{entry.package.name}[/bold]") console.print(f" Catalog: [bold]{catalog.alias}[/bold] ({catalog.url})") - console.print(f" Command: [bold]{shlex.join(plan.command)}[/bold]") + console.print(f" Uninstall target: [bold]{_target_description(plan.uninstall_mode, plan.project_root)}[/bold]") + _display_commands(plan.commands or [plan.command]) if dry_run: print_info("Dry run complete; no changes made") return - if not yes and not confirm_action("Uninstall this package from the current Python environment?", default=False): + if not yes and not confirm_action( + f"Uninstall this package from the {_target_description(plan.uninstall_mode, plan.project_root)}?", + default=False, + ): print_info("No changes made") return @@ -505,6 +518,22 @@ def _display_compatibility(compatibility: CompatibilityResult) -> None: console.print(f" - {reason}") +def _display_commands(commands: list[list[str]]) -> None: + if len(commands) == 1: + console.print(f" Command: [bold]{shlex.join(commands[0])}[/bold]") + return + + console.print(" Commands:") + for command in commands: + console.print(f" [bold]{shlex.join(command)}[/bold]") + + +def _target_description(mode: str, project_root: str | None) -> str: + if mode == "uv-project" and project_root is not None: + return f"current uv project ({project_root})" + return "current Python environment" + + def _format_runtime_plugins(entries: list[PluginCatalogEntry]) -> str: return ", ".join(f"{entry.name} ({entry.plugin_type.value})" for entry in entries) diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index b0cc9d2c3..b9f88018e 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -203,6 +203,15 @@ class CompatibilityResult: reasons: list[str] +@dataclass(frozen=True) +class InstallCommandTemporaryFile: + """Temporary file needed only while executing one install command.""" + + placeholder: str + filename: str + content: str + + @dataclass(frozen=True) class InstallPlan: """Resolved package-manager command for installing one plugin package.""" @@ -214,6 +223,11 @@ class InstallPlan: catalog_alias: str trusted_catalog: bool source_warning: str | None = None + data_designer_protection: str | None = None + command_stdin: str | None = None + temporary_file: InstallCommandTemporaryFile | None = None + install_mode: str = "environment" + project_root: str | None = None @dataclass(frozen=True) @@ -224,6 +238,9 @@ class UninstallPlan: command: list[str] manager: str catalog_alias: str + commands: list[list[str]] | None = None + uninstall_mode: str = "environment" + project_root: str | None = None @dataclass(frozen=True) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 1f5d5ca75..ef9819fd2 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -5,34 +5,72 @@ import importlib import importlib.metadata +import os import shutil import subprocess import sys -from collections.abc import Callable - +import tempfile +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version from data_designer.cli.plugin_catalog import ( + DATA_DESIGNER_DISTRIBUTION_NAME, PLUGIN_ENTRY_POINT_GROUP, PYPI_SIMPLE_INDEX_URL, + InstallCommandTemporaryFile, InstallPlan, PluginCatalogConfig, PluginCatalogEntry, UninstallPlan, ) -InstallRunner = Callable[[list[str]], int] +InstallRunner = Callable[[list[str], str | None], int] PIP_EXTRA_INDEX_SOURCE_WARNING = ( "pip --extra-index-url is not source-pinned; pip may choose a same-named package from another configured index. " "Use uv or a direct reference when strict source selection is required." ) +DATA_DESIGNER_DISTRIBUTION_NAMES = ( + DATA_DESIGNER_DISTRIBUTION_NAME, + "data-designer-config", + "data-designer-engine", +) +DATA_DESIGNER_PROJECT_NAMES = (*DATA_DESIGNER_DISTRIBUTION_NAMES, "data-designer-workspace") +PIP_DATA_DESIGNER_CONSTRAINT_FILE_NAME = "data-designer-constraint.txt" +DATA_DESIGNER_CONSTRAINT_PLACEHOLDER = "" + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - exercised only on Python 3.10. + tomllib = None # type: ignore[assignment] + + +@dataclass(frozen=True) +class _InstallTarget: + manager: str + mode: str + project_root: Path | None = None class PluginInstallService: """Resolve, execute, and verify plugin package install/uninstall plans.""" - def __init__(self, runner: InstallRunner | None = None) -> None: + def __init__( + self, + runner: InstallRunner | None = None, + *, + working_dir: Path | None = None, + active_virtualenv: bool | None = None, + ) -> None: self._runner = runner or _run_subprocess + self._working_dir = working_dir + self._active_virtualenv = active_virtualenv def build_install_plan( self, @@ -42,17 +80,31 @@ def build_install_plan( manager: str = "auto", ) -> InstallPlan: """Build the exact package-manager command for one catalog entry.""" - resolved_manager = _resolve_manager(manager) - install_args, source_description, source_warning = _install_args_for_entry(entry, resolved_manager) - command = _base_command(resolved_manager) + install_args + target = _resolve_install_target( + manager, + working_dir=self._working_dir or Path.cwd(), + active_virtualenv=self._active_virtualenv, + ) + data_designer_version = _installed_data_designer_version() + protection_args, data_designer_protection, command_stdin, temporary_file = _data_designer_protection_args( + target.mode, + data_designer_version, + ) + install_args, source_description, source_warning = _install_args_for_entry(entry, target) + command = _base_command(target) + protection_args + install_args return InstallPlan( package_name=entry.package.name, source_description=source_description, command=command, - manager=resolved_manager, + manager=target.manager, catalog_alias=catalog.alias, trusted_catalog=catalog.trusted, source_warning=source_warning, + data_designer_protection=data_designer_protection, + command_stdin=command_stdin, + temporary_file=temporary_file, + install_mode=target.mode, + project_root=str(target.project_root) if target.project_root is not None else None, ) def build_uninstall_plan( @@ -63,12 +115,20 @@ def build_uninstall_plan( manager: str = "auto", ) -> UninstallPlan: """Build the exact package-manager command to uninstall one catalog package.""" - resolved_manager = _resolve_manager(manager) + target = _resolve_install_target( + manager, + working_dir=self._working_dir or Path.cwd(), + active_virtualenv=self._active_virtualenv, + ) + commands = _uninstall_commands(target, entry.package.name) return UninstallPlan( package_name=entry.package.name, - command=_base_uninstall_command(resolved_manager) + [entry.package.name], - manager=resolved_manager, + command=commands[0], + manager=target.manager, catalog_alias=catalog.alias, + commands=commands, + uninstall_mode=target.mode, + project_root=str(target.project_root) if target.project_root is not None else None, ) def install(self, plan: InstallPlan) -> None: @@ -77,7 +137,8 @@ def install(self, plan: InstallPlan) -> None: Raises: RuntimeError: If the package manager exits unsuccessfully. """ - return_code = self._runner(plan.command) + with _materialized_install_command(plan) as (command, command_stdin): + return_code = self._runner(command, command_stdin) if return_code != 0: raise RuntimeError(f"Plugin package installer exited with status {return_code}") @@ -87,9 +148,10 @@ def uninstall(self, plan: UninstallPlan) -> None: Raises: RuntimeError: If the package manager exits unsuccessfully. """ - return_code = self._runner(plan.command) - if return_code != 0: - raise RuntimeError(f"Plugin package uninstaller exited with status {return_code}") + for command in plan.commands or [plan.command]: + return_code = self._runner(command, None) + if return_code != 0: + raise RuntimeError(f"Plugin package uninstaller exited with status {return_code}") def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: """Verify the plugin's declared entry point is installed.""" @@ -126,8 +188,11 @@ def verify_entry_points_removed(self, entries: list[PluginCatalogEntry]) -> bool ) -def _run_subprocess(command: list[str]) -> int: - result = subprocess.run(command, check=False, stdin=subprocess.DEVNULL) +def _run_subprocess(command: list[str], stdin_text: str | None) -> int: + if stdin_text is None: + result = subprocess.run(command, check=False, stdin=subprocess.DEVNULL) + else: + result = subprocess.run(command, check=False, input=stdin_text, text=True) return result.returncode @@ -161,35 +226,239 @@ def _entry_point_distribution_name(installed_entry_point: importlib.metadata.Ent return name -def _resolve_manager(manager: str) -> str: +def _resolve_install_target( + manager: str, + *, + working_dir: Path, + active_virtualenv: bool | None, +) -> _InstallTarget: if manager not in {"auto", "uv", "pip"}: raise ValueError(f"Unsupported plugin installer {manager!r}. Expected 'auto', 'uv', or 'pip'.") - if manager == "auto": - return "uv" if shutil.which("uv") else "pip" - if manager == "uv" and not shutil.which("uv"): - raise ValueError("uv was requested for plugin package installation, but it is not available on PATH") - return manager + uv_path = shutil.which("uv") if manager in {"auto", "uv"} else None + if manager == "auto": + if uv_path is None: + return _InstallTarget(manager="pip", mode="pip-environment") + project_root = _project_root_for_uv_add(working_dir, active_virtualenv) + if project_root is not None: + return _InstallTarget(manager="uv", mode="uv-project", project_root=project_root) + return _InstallTarget(manager="uv", mode="uv-environment") -def _base_command(manager: str) -> list[str]: if manager == "uv": + if uv_path is None: + raise ValueError("uv was requested for plugin package installation, but it is not available on PATH") + project_root = _project_root_for_uv_add(working_dir, active_virtualenv) + if project_root is not None: + return _InstallTarget(manager="uv", mode="uv-project", project_root=project_root) + return _InstallTarget(manager="uv", mode="uv-environment") + + return _InstallTarget(manager="pip", mode="pip-environment") + + +def _base_command(target: _InstallTarget) -> list[str]: + if target.mode == "uv-project": + if target.project_root is None: + raise ValueError("uv project install target requires a project root") + return ["uv", "add", "--project", str(target.project_root), "--active"] + if target.mode == "uv-environment": return ["uv", "pip", "install", "--python", sys.executable] return [sys.executable, "-m", "pip", "install"] -def _base_uninstall_command(manager: str) -> list[str]: - if manager == "uv": +def _uninstall_commands(target: _InstallTarget, package_name: str) -> list[list[str]]: + if target.mode == "uv-project": + if target.project_root is None: + raise ValueError("uv project uninstall target requires a project root") + return [ + ["uv", "remove", "--project", str(target.project_root), "--no-sync", package_name], + ["uv", "pip", "uninstall", "--python", sys.executable, package_name], + ] + return [_base_uninstall_command(target) + [package_name]] + + +def _base_uninstall_command(target: _InstallTarget) -> list[str]: + if target.manager == "uv": return ["uv", "pip", "uninstall", "--python", sys.executable] return [sys.executable, "-m", "pip", "uninstall", "--yes"] -def _install_args_for_entry(entry: PluginCatalogEntry, manager: str) -> tuple[list[str], str, str | None]: +def _project_root_for_uv_add(working_dir: Path, active_virtualenv: bool | None) -> Path | None: + if not _has_active_virtualenv(active_virtualenv): + return None + + project_root = _find_nearest_pyproject_root(working_dir) + if project_root is None or _is_data_designer_source_project(project_root): + return None + return project_root + + +def _has_active_virtualenv(active_virtualenv: bool | None) -> bool: + if active_virtualenv is not None: + return active_virtualenv + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) or bool(os.getenv("VIRTUAL_ENV")) + + +def _find_nearest_pyproject_root(working_dir: Path) -> Path | None: + resolved_working_dir = working_dir.resolve() + for candidate in (resolved_working_dir, *resolved_working_dir.parents): + if (candidate / "pyproject.toml").is_file(): + return candidate + return None + + +def _is_data_designer_source_project(project_root: Path) -> bool: + pyproject_data = _load_pyproject_data(project_root / "pyproject.toml") + project = pyproject_data.get("project", {}) + if isinstance(project, dict): + project_name = project.get("name") + if isinstance(project_name, str) and canonicalize_name(project_name) in DATA_DESIGNER_PROJECT_NAMES: + return True + + try: + relative_source_file = Path(__file__).resolve().relative_to(project_root.resolve()) + except (OSError, ValueError): + return False + source_parts = relative_source_file.parts + return source_parts[:3] == ("packages", "data-designer", "src") or source_parts[:2] == ("src", "data_designer") + + +def _load_pyproject_data(pyproject_path: Path) -> dict[str, Any]: + try: + text = pyproject_path.read_text(encoding="utf-8") + except OSError: + return {} + + if tomllib is not None: + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError: + return {} + return data if isinstance(data, dict) else {} + + return _load_pyproject_markers_without_tomllib(text) + + +def _load_pyproject_markers_without_tomllib(text: str) -> dict[str, Any]: + project: dict[str, Any] = {} + section = "" + + for raw_line in text.splitlines(): + line = raw_line.split("#", maxsplit=1)[0].strip() + if not line: + continue + if line.startswith("[") and line.endswith("]"): + section = line.strip("[]").strip() + continue + if "=" not in line: + continue + + key, raw_value = (part.strip() for part in line.split("=", maxsplit=1)) + if section == "project" and key == "name": + project["name"] = _parse_simple_toml_value(raw_value) + + data: dict[str, Any] = {} + if project: + data["project"] = project + return data + + +def _parse_simple_toml_value(raw_value: str) -> str | None: + value = raw_value.strip() + if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")): + return value[1:-1] + return None + + +def _installed_data_designer_version() -> str: + try: + version = importlib.metadata.version(DATA_DESIGNER_DISTRIBUTION_NAME) + except importlib.metadata.PackageNotFoundError as e: + raise ValueError( + f"Unable to resolve installed {DATA_DESIGNER_DISTRIBUTION_NAME!r} version; " + "plugin package installs require Data Designer to be installed first." + ) from e + + try: + Version(version) + except InvalidVersion as e: + raise ValueError( + f"Installed {DATA_DESIGNER_DISTRIBUTION_NAME!r} version {version!r} is not a valid package version; " + "cannot protect the current Data Designer installation during plugin package install." + ) from e + return version + + +def _data_designer_protection_args( + mode: str, + version: str, +) -> tuple[list[str], str, str | None, InstallCommandTemporaryFile | None]: + if mode == "uv-environment": + return ( + ["--excludes", "-"], + f"using installed {DATA_DESIGNER_DISTRIBUTION_NAME} {version}; uv will not resolve it", + f"{DATA_DESIGNER_DISTRIBUTION_NAME}\n", + None, + ) + + if mode == "uv-project": + return ( + [ + *[ + item + for distribution_name in DATA_DESIGNER_DISTRIBUTION_NAMES + for item in ("--no-install-package", distribution_name) + ], + ], + f"using installed {DATA_DESIGNER_DISTRIBUTION_NAME} {version}; uv will not install Data Designer packages", + None, + None, + ) + + return ( + ["--constraint", DATA_DESIGNER_CONSTRAINT_PLACEHOLDER], + f"pinned to installed {DATA_DESIGNER_DISTRIBUTION_NAME} {version}", + None, + _data_designer_constraint_file(version), + ) + + +def _data_designer_constraint_file(version: str) -> InstallCommandTemporaryFile: + return InstallCommandTemporaryFile( + placeholder=DATA_DESIGNER_CONSTRAINT_PLACEHOLDER, + filename=PIP_DATA_DESIGNER_CONSTRAINT_FILE_NAME, + content=f"# Data Designer is provided by the active CLI environment.\n" + f"{DATA_DESIGNER_DISTRIBUTION_NAME}=={version}\n", + ) + + +@contextmanager +def _materialized_install_command(plan: InstallPlan) -> Iterator[tuple[list[str], str | None]]: + temporary_file = plan.temporary_file + if temporary_file is None: + yield plan.command, plan.command_stdin + return + + with tempfile.TemporaryDirectory(prefix="data-designer-plugin-install-") as temp_dir: + temporary_path = Path(temp_dir) / temporary_file.filename + temporary_path.write_text(temporary_file.content, encoding="utf-8") + command = [str(temporary_path) if part == temporary_file.placeholder else part for part in plan.command] + yield command, plan.command_stdin + + +def _install_args_for_entry(entry: PluginCatalogEntry, target: _InstallTarget) -> tuple[list[str], str, str | None]: requirement = entry.install.requirement index_url = entry.install.index_url + if target.mode == "uv-project": + args = ["--raw"] if index_url is None and _requirement_is_direct_reference(requirement) else [] + if index_url is not None: + args.extend(["--index", index_url]) + args.append(requirement) + return args, _source_description(requirement, index_url), None + if index_url is None: return [requirement], requirement, None - if manager == "uv": + if target.manager == "uv": return ( ["--default-index", PYPI_SIMPLE_INDEX_URL, "--index", index_url, requirement], f"{requirement} via {index_url}", @@ -200,3 +469,16 @@ def _install_args_for_entry(entry: PluginCatalogEntry, manager: str) -> tuple[li f"{requirement} via {index_url}", PIP_EXTRA_INDEX_SOURCE_WARNING, ) + + +def _source_description(requirement: str, index_url: str | None) -> str: + if index_url is None: + return requirement + return f"{requirement} via {index_url}" + + +def _requirement_is_direct_reference(requirement: str) -> bool: + try: + return Requirement(requirement).url is not None + except InvalidRequirement: + return " @ " in requirement diff --git a/packages/data-designer/tests/cli/commands/test_plugin_command.py b/packages/data-designer/tests/cli/commands/test_plugin_command.py index 5b8cac2fa..1ab244bc7 100644 --- a/packages/data-designer/tests/cli/commands/test_plugin_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugin_command.py @@ -61,7 +61,6 @@ def test_plugin_install_command_delegates_to_controller(mock_ctrl_cls: MagicMock manager="pip", yes=True, dry_run=True, - force=False, ) @@ -102,8 +101,7 @@ def test_plugin_install_help_uses_package_first_wording() -> None: assert "PACKAGE" in result.output assert "Plugin package name or package alias" in result.output assert "runtime plugin name" not in result.output - assert "Allow installing a catalog" in result.output - assert "package when compatibility" in result.output + assert "Print the install plan" in result.output def test_plugin_uninstall_help_uses_package_first_wording() -> None: diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 730ad0b93..8cfdc980b 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -239,7 +239,7 @@ def test_run_install_dry_run_renders_plan_without_installing( ) -> None: entry = _entry() catalog = _catalog(trusted=True) - plan = _plan(catalog) + plan = _plan(catalog, data_designer_protection="pinned to installed data-designer 0.5.10") controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) @@ -257,13 +257,14 @@ def test_run_install_dry_run_renders_plan_without_installing( controller.install_service.install.assert_not_called() controller.install_service.verify_entry_points.assert_not_called() mock_print_info.assert_any_call("Dry run complete; no changes made") + mock_console.print.assert_any_call(" Data Designer: [bold]pinned to installed data-designer 0.5.10[/bold]") assert all("Runtime plugins" not in str(call_args.args[0]) for call_args in mock_console.print.call_args_list) assert mock_console.print.call_count >= 1 @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") -def test_run_install_blocks_incompatible_package_without_force( +def test_run_install_blocks_incompatible_package( mock_print_error: MagicMock, mock_console: MagicMock, controller: PluginCatalogController, @@ -345,15 +346,15 @@ def test_run_install_dry_run_renders_incompatible_plan_and_block_message( mock_console.print.assert_any_call(" Compatibility: [bold yellow]not compatible[/bold yellow]") mock_console.print.assert_any_call(" - Data Designer 0.5.7 does not satisfy >=99.0") mock_print_warning.assert_called_once_with( - "Dry run complete; no changes made. A real install would be blocked unless you pass --force." + "Dry run complete; no changes made. A real install would be blocked because compatibility checks failed." ) @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_error") -@patch("data_designer.cli.controllers.plugin_catalog_controller.print_info") -def test_run_install_force_allows_incompatible_entry_for_dry_run( - mock_print_info: MagicMock, +@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") +def test_run_install_dry_run_allows_incompatible_entry_for_inspection( + mock_print_warning: MagicMock, mock_print_error: MagicMock, mock_console: MagicMock, controller: PluginCatalogController, @@ -368,7 +369,7 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( ) controller.install_service.build_install_plan.return_value = _plan(catalog) - controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True, force=True) + controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( @@ -380,7 +381,9 @@ def test_run_install_force_allows_incompatible_entry_for_dry_run( controller.install_service.build_install_plan.assert_called_once_with(entry, catalog, manager="auto") controller.install_service.install.assert_not_called() mock_print_error.assert_not_called() - mock_print_info.assert_any_call("Dry run complete; no changes made") + mock_print_warning.assert_called_once_with( + "Dry run complete; no changes made. A real install would be blocked because compatibility checks failed." + ) assert mock_console.print.call_count >= 1 @@ -625,7 +628,12 @@ def _catalog(*, trusted: bool) -> PluginCatalogConfig: ) -def _plan(catalog: PluginCatalogConfig, *, source_warning: str | None = None) -> InstallPlan: +def _plan( + catalog: PluginCatalogConfig, + *, + source_warning: str | None = None, + data_designer_protection: str | None = None, +) -> InstallPlan: return InstallPlan( package_name="data-designer-text-transform", source_description="data-designer-text-transform", @@ -634,6 +642,7 @@ def _plan(catalog: PluginCatalogConfig, *, source_warning: str | None = None) -> catalog_alias=catalog.alias, trusted_catalog=catalog.trusted, source_warning=source_warning, + data_designer_protection=data_designer_protection, ) diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index 940c129e9..308f3a6af 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -3,7 +3,10 @@ from __future__ import annotations +import importlib.metadata import sys +from collections.abc import Iterator +from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock, patch @@ -12,6 +15,17 @@ from data_designer.cli.plugin_catalog import PluginCatalogConfig, PluginCatalogEntry from data_designer.cli.services.plugin_install_service import PIP_EXTRA_INDEX_SOURCE_WARNING, PluginInstallService +DATA_DESIGNER_VERSION = "0.5.10" + + +@pytest.fixture(autouse=True) +def mock_data_designer_version() -> Iterator[None]: + with patch( + "data_designer.cli.services.plugin_install_service.importlib.metadata.version", + return_value=DATA_DESIGNER_VERSION, + ): + yield + def test_build_pip_install_plan_uses_requirement_and_extra_index() -> None: entry = _entry( @@ -33,10 +47,19 @@ def test_build_pip_install_plan_uses_requirement_and_extra_index() -> None: "-m", "pip", "install", + "--constraint", + "", "--extra-index-url", "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", "data-designer-template", ] + assert plan.temporary_file is not None + assert plan.temporary_file.filename == "data-designer-constraint.txt" + assert plan.temporary_file.content == ( + f"# Data Designer is provided by the active CLI environment.\ndata-designer=={DATA_DESIGNER_VERSION}\n" + ) + assert plan.command_stdin is None + assert plan.data_designer_protection == f"pinned to installed data-designer {DATA_DESIGNER_VERSION}" assert plan.source_description == ( "data-designer-template via https://nvidia-nemo.github.io/DataDesignerPlugins/simple/" ) @@ -76,22 +99,135 @@ def test_build_auto_install_plan_chooses_uv_when_available(mock_which: Mock) -> plan = service.build_install_plan(entry, catalog, manager="auto") assert plan.manager == "uv" + assert plan.install_mode == "uv-environment" assert plan.command == [ "uv", "pip", "install", "--python", sys.executable, + "--excludes", + "-", "--default-index", "https://pypi.org/simple/", "--index", "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", "data-designer-template", ] + assert plan.command_stdin == "data-designer\n" + assert plan.temporary_file is None + assert ( + plan.data_designer_protection + == f"using installed data-designer {DATA_DESIGNER_VERSION}; uv will not resolve it" + ) + assert plan.source_warning is None + mock_which.assert_called_once_with("uv") + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_uses_uv_add_for_active_project(mock_which: Mock, tmp_path: Path) -> None: + working_dir = _write_project(tmp_path) / "src" + working_dir.mkdir() + entry = _entry( + package_name="data-designer-template", + install={ + "requirement": "data-designer-template", + "index_url": "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + }, + ) + catalog = PluginCatalogConfig( + alias="nvidia", url="https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json" + ) + service = PluginInstallService(working_dir=working_dir, active_virtualenv=True) + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.manager == "uv" + assert plan.install_mode == "uv-project" + assert plan.project_root == str(tmp_path) + assert plan.command == [ + "uv", + "add", + "--project", + str(tmp_path), + "--active", + "--no-install-package", + "data-designer", + "--no-install-package", + "data-designer-config", + "--no-install-package", + "data-designer-engine", + "--index", + "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", + "data-designer-template", + ] + assert plan.command_stdin is None + assert plan.temporary_file is None + assert ( + plan.data_designer_protection + == f"using installed data-designer {DATA_DESIGNER_VERSION}; uv will not install Data Designer packages" + ) assert plan.source_warning is None mock_which.assert_called_once_with("uv") +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_does_not_use_uv_add_without_active_virtualenv( + mock_which: Mock, + tmp_path: Path, +) -> None: + _write_project(tmp_path) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService(working_dir=tmp_path, active_virtualenv=False) + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.install_mode == "uv-environment" + assert plan.command[:6] == ["uv", "pip", "install", "--python", sys.executable, "--excludes"] + mock_which.assert_called_once_with("uv") + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_does_not_use_uv_add_for_data_designer_workspace( + mock_which: Mock, + tmp_path: Path, +) -> None: + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "data-designer-workspace"\n[tool.uv]\npackage = false\n', + encoding="utf-8", + ) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService(working_dir=tmp_path, active_virtualenv=True) + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.install_mode == "uv-environment" + assert plan.project_root is None + mock_which.assert_called_once_with("uv") + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_uses_uv_add_for_non_package_user_project( + mock_which: Mock, + tmp_path: Path, +) -> None: + (tmp_path / "pyproject.toml").write_text( + '[project]\nname = "experiment-workspace"\n[tool.uv]\npackage = false\n', + encoding="utf-8", + ) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService(working_dir=tmp_path, active_virtualenv=True) + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.install_mode == "uv-project" + assert plan.project_root == str(tmp_path) + mock_which.assert_called_once_with("uv") + + @patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value=None) def test_build_auto_install_plan_chooses_pip_when_uv_is_unavailable(mock_which: Mock) -> None: entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) @@ -101,7 +237,16 @@ def test_build_auto_install_plan_chooses_pip_when_uv_is_unavailable(mock_which: plan = service.build_install_plan(entry, catalog, manager="auto") assert plan.manager == "pip" - assert plan.command == [sys.executable, "-m", "pip", "install", "data-designer-template"] + assert plan.command == [ + sys.executable, + "-m", + "pip", + "install", + "--constraint", + "", + "data-designer-template", + ] + assert plan.temporary_file is not None mock_which.assert_called_once_with("uv") @@ -145,6 +290,47 @@ def test_build_auto_uninstall_plan_chooses_uv_when_available(mock_which: Mock) - "data-designer-template", ] assert plan.manager == "uv" + assert plan.uninstall_mode == "uv-environment" + mock_which.assert_called_once_with("uv") + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_uninstall_plan_uses_uv_remove_for_active_project(mock_which: Mock, tmp_path: Path) -> None: + _write_project(tmp_path) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService(working_dir=tmp_path, active_virtualenv=True) + + plan = service.build_uninstall_plan(entry, catalog, manager="auto") + + assert plan.command == [ + "uv", + "remove", + "--project", + str(tmp_path), + "--no-sync", + "data-designer-template", + ] + assert plan.commands == [ + [ + "uv", + "remove", + "--project", + str(tmp_path), + "--no-sync", + "data-designer-template", + ], + [ + "uv", + "pip", + "uninstall", + "--python", + sys.executable, + "data-designer-template", + ], + ] + assert plan.uninstall_mode == "uv-project" + assert plan.project_root == str(tmp_path) mock_which.assert_called_once_with("uv") @@ -170,12 +356,34 @@ def test_build_uv_install_plan_targets_current_python_and_adds_catalog_index(moc "install", "--python", sys.executable, + "--excludes", + "-", "--default-index", "https://pypi.org/simple/", "--index", "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", "data-designer-template", ] + assert plan.command_stdin == "data-designer\n" + assert plan.temporary_file is None + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_uv_add_plan_preserves_direct_reference_with_raw(mock_which: Mock, tmp_path: Path) -> None: + _write_project(tmp_path) + requirement = ( + "data-designer-template @ " + "git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git@data-designer-template/v0.1.0" + ) + entry = _entry(package_name="data-designer-template", install={"requirement": requirement}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService(working_dir=tmp_path, active_virtualenv=True) + + plan = service.build_install_plan(entry, catalog, manager="uv") + + assert plan.command[-2:] == ["--raw", requirement] + assert "--index" not in plan.command + mock_which.assert_called_once_with("uv") @patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value=None) @@ -190,8 +398,38 @@ def test_build_uv_install_plan_raises_when_uv_is_unavailable(mock_which: Mock) - mock_which.assert_called_once_with("uv") +def test_build_install_plan_requires_installed_data_designer_version() -> None: + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + with ( + patch( + "data_designer.cli.services.plugin_install_service.importlib.metadata.version", + side_effect=importlib.metadata.PackageNotFoundError, + ), + pytest.raises(ValueError, match="Unable to resolve installed 'data-designer' version"), + ): + service.build_install_plan(entry, catalog, manager="pip") + + +def test_build_install_plan_rejects_invalid_installed_data_designer_version() -> None: + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + with ( + patch( + "data_designer.cli.services.plugin_install_service.importlib.metadata.version", + return_value="not a version", + ), + pytest.raises(ValueError, match="version 'not a version' is not a valid package version"), + ): + service.build_install_plan(entry, catalog, manager="pip") + + def test_install_raises_when_runner_fails() -> None: - service = PluginInstallService(runner=lambda command: 2) + service = PluginInstallService(runner=lambda command, stdin_text: 2) entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") plan = service.build_install_plan(entry, catalog, manager="pip") @@ -200,8 +438,68 @@ def test_install_raises_when_runner_fails() -> None: service.install(plan) +def test_install_materializes_pip_constraint_as_temporary_file() -> None: + seen: dict[str, Path | str | None] = {} + + def runner(command: list[str], stdin_text: str | None) -> int: + constraint_file = Path(command[command.index("--constraint") + 1]) + seen["constraint_file"] = constraint_file + seen["constraint_parent"] = constraint_file.parent + seen["constraint_text"] = constraint_file.read_text(encoding="utf-8") + seen["stdin_text"] = stdin_text + return 0 + + service = PluginInstallService(runner=runner) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + plan = service.build_install_plan(entry, catalog, manager="pip") + + service.install(plan) + + constraint_file = seen["constraint_file"] + constraint_parent = seen["constraint_parent"] + assert isinstance(constraint_file, Path) + assert isinstance(constraint_parent, Path) + assert not constraint_file.exists() + assert not constraint_parent.exists() + assert seen["constraint_text"] == ( + f"# Data Designer is provided by the active CLI environment.\ndata-designer=={DATA_DESIGNER_VERSION}\n" + ) + assert seen["stdin_text"] is None + + +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_install_passes_uv_exclude_over_stdin(mock_which: Mock) -> None: + seen: dict[str, list[str] | str | None] = {} + + def runner(command: list[str], stdin_text: str | None) -> int: + seen["command"] = command + seen["stdin_text"] = stdin_text + return 0 + + service = PluginInstallService(runner=runner) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + plan = service.build_install_plan(entry, catalog, manager="uv") + + service.install(plan) + + assert seen["command"] == [ + "uv", + "pip", + "install", + "--python", + sys.executable, + "--excludes", + "-", + "data-designer-template", + ] + assert seen["stdin_text"] == "data-designer\n" + mock_which.assert_called_once_with("uv") + + def test_uninstall_raises_when_runner_fails() -> None: - service = PluginInstallService(runner=lambda command: 2) + service = PluginInstallService(runner=lambda command, stdin_text: 2) entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") plan = service.build_uninstall_plan(entry, catalog, manager="pip") @@ -210,6 +508,27 @@ def test_uninstall_raises_when_runner_fails() -> None: service.uninstall(plan) +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_uninstall_runs_every_project_uninstall_command(mock_which: Mock, tmp_path: Path) -> None: + seen: list[list[str]] = [] + + def runner(command: list[str], stdin_text: str | None) -> int: + assert stdin_text is None + seen.append(command) + return 0 + + _write_project(tmp_path) + service = PluginInstallService(runner=runner, working_dir=tmp_path, active_virtualenv=True) + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + plan = service.build_uninstall_plan(entry, catalog, manager="auto") + + service.uninstall(plan) + + assert seen == plan.commands + mock_which.assert_called_once_with("uv") + + @patch("data_designer.cli.services.plugin_install_service.importlib.metadata.entry_points") @patch("data_designer.cli.services.plugin_install_service.importlib.invalidate_caches") def test_verify_entry_point_invalidates_caches_and_checks_declared_entry_point( @@ -413,3 +732,9 @@ def _entry( }, } return PluginCatalogEntry.model_validate(payload) + + +def _write_project(path: Path, *, name: str = "synthetic-data-project") -> Path: + path.mkdir(exist_ok=True) + (path / "pyproject.toml").write_text(f'[project]\nname = "{name}"\n', encoding="utf-8") + return path From ef8166c03c6ea55041f7c48fd994b43cfe16f328 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Fri, 8 May 2026 22:37:51 +0000 Subject: [PATCH 23/34] document plugin package install modes --- architecture/cli.md | 10 ++++++++-- .../data-designer/src/data_designer/cli/README.md | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/architecture/cli.md b/architecture/cli.md index 405c7d4bd..9ed703626 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -87,14 +87,20 @@ User invokes command (e.g., `data-designer plugin list`) User invokes command (e.g., `data-designer plugin install calculator`) → PluginCatalogController resolves the plugin package name or package alias → PluginCatalogService evaluates Python and Data Designer compatibility - → PluginInstallService builds a pip/uv install plan for the package requirement + → PluginInstallService builds a pip/uv install plan for the package requirement. + In active uv projects it uses `uv add` so the package lands in `pyproject.toml`; + otherwise it mutates the active Python environment with `uv pip install` or pip. + The active `data-designer` distribution is skipped, excluded, or pinned from + replacement depending on package-manager capabilities. → PluginInstallService verifies declared package entry points after installation ``` ``` User invokes command (e.g., `data-designer plugin uninstall calculator`) → PluginCatalogController resolves the plugin package name or package alias - → PluginInstallService builds a pip/uv uninstall plan for the package distribution + → PluginInstallService builds a pip/uv uninstall plan for the package distribution. + Active uv projects remove the dependency from project metadata without a uv sync, + then uninstall the package from the active environment. → PluginInstallService verifies declared package entry points are no longer discovered ``` diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index bc318da22..cd2c3fb1d 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -8,7 +8,7 @@ The CLI provides an interactive interface for managing: - **Model Providers**: LLM API endpoints (NVIDIA, OpenAI, Anthropic, custom providers) - **Model Configs**: Specific model configurations with inference parameters - **Plugin Catalogs**: Catalog aliases for discovering Data Designer plugin packages -- **Plugin Installs**: Safe install-plan rendering, package-manager execution, and entry point verification +- **Plugin Installs**: Safe install-plan rendering, package-manager execution with active Data Designer protection, and entry point verification Configuration files are stored in `~/.data-designer/` by default and can be referenced by Data Designer workflows. @@ -84,7 +84,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `model_service.py`: Model configuration business logic - `provider_service.py`: Provider business logic - `plugin_catalog_service.py`: Plugin catalog discovery, search, compatibility checks, and installed entry point listing - - `plugin_install_service.py`: Plugin install/uninstall plan resolution, package-manager execution, and runtime verification + - `plugin_install_service.py`: Plugin install/uninstall plan resolution, package-manager execution, active Data Designer protection, and runtime verification **Key Methods**: - `list_all()`: Get all configured items @@ -313,3 +313,13 @@ data-designer plugin catalog remove research # List installed runtime plugin entry points without importing plugin modules data-designer plugin installed ``` + +Install plans protect the active `data-designer` distribution before invoking +the package manager. The plugin package and its other dependencies are resolved +normally, but `data-designer` itself is kept from being replaced by the plugin +package dependency. In an active virtual environment with a user `pyproject.toml`, +`uv` uses `uv add` so the plugin package is recorded in the project; otherwise it +uses `uv pip install`. `pip` remains supported for pip-only environments. `uv` +project installs skip installing the Data Designer package family; pip installs +use a process-scoped temporary constraint file because pip constraints are +file-based. From 6cfa3ef5ea66bcf6a4b85597abc73bb1c6bc6466 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 00:05:16 +0000 Subject: [PATCH 24/34] avoid building project during plugin installs --- .../src/data_designer/cli/services/plugin_install_service.py | 2 +- .../tests/cli/services/test_plugin_install_service.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index ef9819fd2..293af73eb 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -259,7 +259,7 @@ def _base_command(target: _InstallTarget) -> list[str]: if target.mode == "uv-project": if target.project_root is None: raise ValueError("uv project install target requires a project root") - return ["uv", "add", "--project", str(target.project_root), "--active"] + return ["uv", "add", "--project", str(target.project_root), "--active", "--no-install-project"] if target.mode == "uv-environment": return ["uv", "pip", "install", "--python", sys.executable] return [sys.executable, "-m", "pip", "install"] diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index 308f3a6af..516547514 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -151,6 +151,7 @@ def test_build_auto_install_plan_uses_uv_add_for_active_project(mock_which: Mock "--project", str(tmp_path), "--active", + "--no-install-project", "--no-install-package", "data-designer", "--no-install-package", From 05bfaf09b2b0d743876a678109220be95c4035b6 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 00:29:59 +0000 Subject: [PATCH 25/34] harden plugin package installs --- .../src/data_designer/cli/README.md | 20 +-- .../cli/services/plugin_install_service.py | 128 ++++++++++++++---- .../services/test_plugin_install_service.py | 92 +++++++++++-- 3 files changed, 192 insertions(+), 48 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index cd2c3fb1d..9899cb9ed 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -314,12 +314,14 @@ data-designer plugin catalog remove research data-designer plugin installed ``` -Install plans protect the active `data-designer` distribution before invoking -the package manager. The plugin package and its other dependencies are resolved -normally, but `data-designer` itself is kept from being replaced by the plugin -package dependency. In an active virtual environment with a user `pyproject.toml`, -`uv` uses `uv add` so the plugin package is recorded in the project; otherwise it -uses `uv pip install`. `pip` remains supported for pip-only environments. `uv` -project installs skip installing the Data Designer package family; pip installs -use a process-scoped temporary constraint file because pip constraints are -file-based. +Install plans protect the active Data Designer package family (`data-designer`, +`data-designer-config`, and `data-designer-engine`) before invoking the package +manager. The plugin package and its other dependencies are resolved normally, +but the installed Data Designer packages are kept from being replaced by plugin +package dependencies. In an active virtual environment with a user +`pyproject.toml`, `uv` uses `uv add` so the plugin package is recorded in the +project; otherwise it uses `uv pip install`. `uv` plugin installs require +`uv >= 0.6.0`; auto mode falls back to `pip` when `uv` is missing or too old. +`pip` remains supported for pip-only environments. `uv` project installs skip +installing the Data Designer package family; pip installs use a process-scoped +temporary constraint file because pip constraints are file-based. diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 293af73eb..209e16e6a 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -44,6 +44,7 @@ DATA_DESIGNER_PROJECT_NAMES = (*DATA_DESIGNER_DISTRIBUTION_NAMES, "data-designer-workspace") PIP_DATA_DESIGNER_CONSTRAINT_FILE_NAME = "data-designer-constraint.txt" DATA_DESIGNER_CONSTRAINT_PLACEHOLDER = "" +UV_PLUGIN_INSTALL_MIN_VERSION = Version("0.6.0") try: import tomllib @@ -56,10 +57,15 @@ class _InstallTarget: manager: str mode: str project_root: Path | None = None + warning: str | None = None class PluginInstallService: - """Resolve, execute, and verify plugin package install/uninstall plans.""" + """Resolve, execute, and verify plugin package install/uninstall plans. + + When no working directory is provided, plan resolution uses the current + process directory at build time so CLI calls follow the user's active shell. + """ def __init__( self, @@ -85,10 +91,10 @@ def build_install_plan( working_dir=self._working_dir or Path.cwd(), active_virtualenv=self._active_virtualenv, ) - data_designer_version = _installed_data_designer_version() + data_designer_versions = _installed_data_designer_distribution_versions() protection_args, data_designer_protection, command_stdin, temporary_file = _data_designer_protection_args( target.mode, - data_designer_version, + data_designer_versions, ) install_args, source_description, source_warning = _install_args_for_entry(entry, target) command = _base_command(target) + protection_args + install_args @@ -99,7 +105,7 @@ def build_install_plan( manager=target.manager, catalog_alias=catalog.alias, trusted_catalog=catalog.trusted, - source_warning=source_warning, + source_warning=_combine_warnings(target.warning, source_warning), data_designer_protection=data_designer_protection, command_stdin=command_stdin, temporary_file=temporary_file, @@ -236,9 +242,16 @@ def _resolve_install_target( raise ValueError(f"Unsupported plugin installer {manager!r}. Expected 'auto', 'uv', or 'pip'.") uv_path = shutil.which("uv") if manager in {"auto", "uv"} else None + uv_error = _uv_plugin_install_error(uv_path) if uv_path is not None else None if manager == "auto": if uv_path is None: return _InstallTarget(manager="pip", mode="pip-environment") + if uv_error is not None: + return _InstallTarget( + manager="pip", + mode="pip-environment", + warning=f"{uv_error}; falling back to pip.", + ) project_root = _project_root_for_uv_add(working_dir, active_virtualenv) if project_root is not None: return _InstallTarget(manager="uv", mode="uv-project", project_root=project_root) @@ -247,6 +260,8 @@ def _resolve_install_target( if manager == "uv": if uv_path is None: raise ValueError("uv was requested for plugin package installation, but it is not available on PATH") + if uv_error is not None: + raise ValueError(f"{uv_error}. Use --manager pip or update uv.") project_root = _project_root_for_uv_add(working_dir, active_virtualenv) if project_root is not None: return _InstallTarget(manager="uv", mode="uv-project", project_root=project_root) @@ -298,6 +313,44 @@ def _has_active_virtualenv(active_virtualenv: bool | None) -> bool: return sys.prefix != getattr(sys, "base_prefix", sys.prefix) or bool(os.getenv("VIRTUAL_ENV")) +def _uv_plugin_install_error(uv_path: str) -> str | None: + try: + result = subprocess.run( + [uv_path, "--version"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired) as e: + return f"Unable to verify uv at {uv_path!r}: {e}" + + output = (result.stdout or result.stderr).strip() + if result.returncode != 0: + details = f": {output}" if output else "" + return f"Unable to verify uv at {uv_path!r}; `uv --version` exited with status {result.returncode}{details}" + + uv_version = _parse_uv_version(output) + if uv_version is None: + return ( + f"Unable to parse uv version from {output!r}; plugin package installs require " + f"uv >= {UV_PLUGIN_INSTALL_MIN_VERSION}" + ) + if uv_version < UV_PLUGIN_INSTALL_MIN_VERSION: + return f"Found uv {uv_version}, but plugin package installs require uv >= {UV_PLUGIN_INSTALL_MIN_VERSION}" + return None + + +def _parse_uv_version(output: str) -> Version | None: + for token in output.split(): + try: + return Version(token) + except InvalidVersion: + continue + return None + + def _find_nearest_pyproject_root(working_dir: Path) -> Path | None: resolved_working_dir = working_dir.resolve() for candidate in (resolved_working_dir, *resolved_working_dir.parents): @@ -339,6 +392,8 @@ def _load_pyproject_data(pyproject_path: Path) -> dict[str, Any]: def _load_pyproject_markers_without_tomllib(text: str) -> dict[str, Any]: + # Python 3.10 only needs a deliberately lossy fallback: detect simple + # [project] name markers from this repo's pyprojects, not parse TOML. project: dict[str, Any] = {} section = "" @@ -369,34 +424,39 @@ def _parse_simple_toml_value(raw_value: str) -> str | None: return None -def _installed_data_designer_version() -> str: - try: - version = importlib.metadata.version(DATA_DESIGNER_DISTRIBUTION_NAME) - except importlib.metadata.PackageNotFoundError as e: - raise ValueError( - f"Unable to resolve installed {DATA_DESIGNER_DISTRIBUTION_NAME!r} version; " - "plugin package installs require Data Designer to be installed first." - ) from e +def _installed_data_designer_distribution_versions() -> dict[str, str]: + versions: dict[str, str] = {} + for distribution_name in DATA_DESIGNER_DISTRIBUTION_NAMES: + try: + version = importlib.metadata.version(distribution_name) + except importlib.metadata.PackageNotFoundError as e: + raise ValueError( + f"Unable to resolve installed {distribution_name!r} version; " + "plugin package installs require the Data Designer package family to be installed first." + ) from e - try: - Version(version) - except InvalidVersion as e: - raise ValueError( - f"Installed {DATA_DESIGNER_DISTRIBUTION_NAME!r} version {version!r} is not a valid package version; " - "cannot protect the current Data Designer installation during plugin package install." - ) from e - return version + try: + Version(version) + except InvalidVersion as e: + raise ValueError( + f"Installed {distribution_name!r} version {version!r} is not a valid package version; " + "cannot protect the current Data Designer installation during plugin package install." + ) from e + versions[distribution_name] = version + return versions def _data_designer_protection_args( mode: str, - version: str, + versions: dict[str, str], ) -> tuple[list[str], str, str | None, InstallCommandTemporaryFile | None]: + data_designer_version = versions[DATA_DESIGNER_DISTRIBUTION_NAME] if mode == "uv-environment": return ( ["--excludes", "-"], - f"using installed {DATA_DESIGNER_DISTRIBUTION_NAME} {version}; uv will not resolve it", - f"{DATA_DESIGNER_DISTRIBUTION_NAME}\n", + f"using installed {DATA_DESIGNER_DISTRIBUTION_NAME} {data_designer_version}; " + "uv will not resolve Data Designer packages", + "".join(f"{distribution_name}\n" for distribution_name in DATA_DESIGNER_DISTRIBUTION_NAMES), None, ) @@ -409,25 +469,28 @@ def _data_designer_protection_args( for item in ("--no-install-package", distribution_name) ], ], - f"using installed {DATA_DESIGNER_DISTRIBUTION_NAME} {version}; uv will not install Data Designer packages", + f"using installed {DATA_DESIGNER_DISTRIBUTION_NAME} {data_designer_version}; " + "uv will not install Data Designer packages", None, None, ) return ( ["--constraint", DATA_DESIGNER_CONSTRAINT_PLACEHOLDER], - f"pinned to installed {DATA_DESIGNER_DISTRIBUTION_NAME} {version}", + f"pinned installed Data Designer packages; {DATA_DESIGNER_DISTRIBUTION_NAME} {data_designer_version}", None, - _data_designer_constraint_file(version), + _data_designer_constraint_file(versions), ) -def _data_designer_constraint_file(version: str) -> InstallCommandTemporaryFile: +def _data_designer_constraint_file(versions: dict[str, str]) -> InstallCommandTemporaryFile: + constraints = "\n".join( + f"{distribution_name}=={versions[distribution_name]}" for distribution_name in DATA_DESIGNER_DISTRIBUTION_NAMES + ) return InstallCommandTemporaryFile( placeholder=DATA_DESIGNER_CONSTRAINT_PLACEHOLDER, filename=PIP_DATA_DESIGNER_CONSTRAINT_FILE_NAME, - content=f"# Data Designer is provided by the active CLI environment.\n" - f"{DATA_DESIGNER_DISTRIBUTION_NAME}=={version}\n", + content=f"# Data Designer is provided by the active CLI environment.\n{constraints}\n", ) @@ -477,6 +540,13 @@ def _source_description(requirement: str, index_url: str | None) -> str: return f"{requirement} via {index_url}" +def _combine_warnings(*warnings: str | None) -> str | None: + active_warnings = [warning for warning in warnings if warning] + if not active_warnings: + return None + return "\n".join(active_warnings) + + def _requirement_is_direct_reference(requirement: str) -> bool: try: return Requirement(requirement).url is not None diff --git a/packages/data-designer/tests/cli/services/test_plugin_install_service.py b/packages/data-designer/tests/cli/services/test_plugin_install_service.py index 516547514..5fe5d163a 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_install_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_install_service.py @@ -20,9 +20,15 @@ @pytest.fixture(autouse=True) def mock_data_designer_version() -> Iterator[None]: - with patch( - "data_designer.cli.services.plugin_install_service.importlib.metadata.version", - return_value=DATA_DESIGNER_VERSION, + with ( + patch( + "data_designer.cli.services.plugin_install_service.importlib.metadata.version", + return_value=DATA_DESIGNER_VERSION, + ), + patch( + "data_designer.cli.services.plugin_install_service.subprocess.run", + return_value=SimpleNamespace(returncode=0, stdout="uv 0.6.0\n", stderr=""), + ), ): yield @@ -56,10 +62,16 @@ def test_build_pip_install_plan_uses_requirement_and_extra_index() -> None: assert plan.temporary_file is not None assert plan.temporary_file.filename == "data-designer-constraint.txt" assert plan.temporary_file.content == ( - f"# Data Designer is provided by the active CLI environment.\ndata-designer=={DATA_DESIGNER_VERSION}\n" + "# Data Designer is provided by the active CLI environment.\n" + f"data-designer=={DATA_DESIGNER_VERSION}\n" + f"data-designer-config=={DATA_DESIGNER_VERSION}\n" + f"data-designer-engine=={DATA_DESIGNER_VERSION}\n" ) assert plan.command_stdin is None - assert plan.data_designer_protection == f"pinned to installed data-designer {DATA_DESIGNER_VERSION}" + assert ( + plan.data_designer_protection + == f"pinned installed Data Designer packages; data-designer {DATA_DESIGNER_VERSION}" + ) assert plan.source_description == ( "data-designer-template via https://nvidia-nemo.github.io/DataDesignerPlugins/simple/" ) @@ -114,11 +126,11 @@ def test_build_auto_install_plan_chooses_uv_when_available(mock_which: Mock) -> "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", "data-designer-template", ] - assert plan.command_stdin == "data-designer\n" + assert plan.command_stdin == "data-designer\ndata-designer-config\ndata-designer-engine\n" assert plan.temporary_file is None assert ( plan.data_designer_protection - == f"using installed data-designer {DATA_DESIGNER_VERSION}; uv will not resolve it" + == f"using installed data-designer {DATA_DESIGNER_VERSION}; uv will not resolve Data Designer packages" ) assert plan.source_warning is None mock_which.assert_called_once_with("uv") @@ -209,6 +221,23 @@ def test_build_auto_install_plan_does_not_use_uv_add_for_data_designer_workspace mock_which.assert_called_once_with("uv") +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_does_not_use_uv_add_for_active_virtualenv_without_pyproject( + mock_which: Mock, + tmp_path: Path, +) -> None: + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService(working_dir=tmp_path, active_virtualenv=True) + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.manager == "uv" + assert plan.install_mode == "uv-environment" + assert plan.project_root is None + mock_which.assert_called_once_with("uv") + + @patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") def test_build_auto_install_plan_uses_uv_add_for_non_package_user_project( mock_which: Mock, @@ -251,6 +280,28 @@ def test_build_auto_install_plan_chooses_pip_when_uv_is_unavailable(mock_which: mock_which.assert_called_once_with("uv") +@patch("data_designer.cli.services.plugin_install_service._uv_plugin_install_error") +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_auto_install_plan_falls_back_to_pip_when_uv_is_too_old( + mock_which: Mock, + mock_uv_error: Mock, +) -> None: + mock_uv_error.return_value = "Found uv 0.5.0, but plugin package installs require uv >= 0.6.0" + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + plan = service.build_install_plan(entry, catalog, manager="auto") + + assert plan.manager == "pip" + assert plan.install_mode == "pip-environment" + assert plan.source_warning == ( + "Found uv 0.5.0, but plugin package installs require uv >= 0.6.0; falling back to pip." + ) + mock_which.assert_called_once_with("uv") + mock_uv_error.assert_called_once_with("/usr/bin/uv") + + def test_build_pip_uninstall_plan_uses_package_name_not_install_requirement() -> None: requirement = ( "data-designer-template @ " @@ -365,7 +416,7 @@ def test_build_uv_install_plan_targets_current_python_and_adds_catalog_index(moc "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/", "data-designer-template", ] - assert plan.command_stdin == "data-designer\n" + assert plan.command_stdin == "data-designer\ndata-designer-config\ndata-designer-engine\n" assert plan.temporary_file is None @@ -399,6 +450,24 @@ def test_build_uv_install_plan_raises_when_uv_is_unavailable(mock_which: Mock) - mock_which.assert_called_once_with("uv") +@patch("data_designer.cli.services.plugin_install_service._uv_plugin_install_error") +@patch("data_designer.cli.services.plugin_install_service.shutil.which", return_value="/usr/bin/uv") +def test_build_uv_install_plan_raises_when_uv_is_too_old( + mock_which: Mock, + mock_uv_error: Mock, +) -> None: + mock_uv_error.return_value = "Found uv 0.5.0, but plugin package installs require uv >= 0.6.0" + entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) + catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") + service = PluginInstallService() + + with pytest.raises(ValueError, match="plugin package installs require uv >= 0.6.0"): + service.build_install_plan(entry, catalog, manager="uv") + + mock_which.assert_called_once_with("uv") + mock_uv_error.assert_called_once_with("/usr/bin/uv") + + def test_build_install_plan_requires_installed_data_designer_version() -> None: entry = _entry(package_name="data-designer-template", install={"requirement": "data-designer-template"}) catalog = PluginCatalogConfig(alias="local", url="/catalog/plugins.json") @@ -464,7 +533,10 @@ def runner(command: list[str], stdin_text: str | None) -> int: assert not constraint_file.exists() assert not constraint_parent.exists() assert seen["constraint_text"] == ( - f"# Data Designer is provided by the active CLI environment.\ndata-designer=={DATA_DESIGNER_VERSION}\n" + "# Data Designer is provided by the active CLI environment.\n" + f"data-designer=={DATA_DESIGNER_VERSION}\n" + f"data-designer-config=={DATA_DESIGNER_VERSION}\n" + f"data-designer-engine=={DATA_DESIGNER_VERSION}\n" ) assert seen["stdin_text"] is None @@ -495,7 +567,7 @@ def runner(command: list[str], stdin_text: str | None) -> int: "-", "data-designer-template", ] - assert seen["stdin_text"] == "data-designer\n" + assert seen["stdin_text"] == "data-designer\ndata-designer-config\ndata-designer-engine\n" mock_which.assert_called_once_with("uv") From 2e64d142cff45acb19ef4c5cd43480cc3e5203ab Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 00:30:03 +0000 Subject: [PATCH 26/34] tighten plugin catalog contracts --- .../src/data_designer/cli/plugin_catalog.py | 24 ++- .../repositories/plugin_catalog_repository.py | 12 +- .../cli/services/plugin_catalog_service.py | 34 +-- .../test_plugin_catalog_controller.py | 13 +- .../catalog-invalid-install.json | 36 ++++ .../catalog-unsupported-version.json | 4 + .../upstream-catalogs/catalog-valid.json | 204 ++++++++++++++++++ .../test_plugin_catalog_repository.py | 62 ++++++ .../services/test_plugin_catalog_service.py | 68 +----- 9 files changed, 340 insertions(+), 117 deletions(-) create mode 100644 packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-invalid-install.json create mode 100644 packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-unsupported-version.json create mode 100644 packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-valid.json diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index b9f88018e..07b5d273d 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -58,7 +58,7 @@ class PluginCompatibilityTarget(BaseModel): model_config = ConfigDict(extra="forbid") requirement: str | None = None - specifier: str | None = None + specifier: str = Field(min_length=1) marker: str | None = None @@ -67,8 +67,8 @@ class PluginCompatibility(BaseModel): model_config = ConfigDict(extra="forbid") - python: PluginCompatibilityTarget | None = None - data_designer: PluginCompatibilityTarget | None = None + python: PluginCompatibilityTarget + data_designer: PluginCompatibilityTarget class PluginPackageInfo(BaseModel): @@ -103,7 +103,7 @@ class PluginDocsInfo(BaseModel): model_config = ConfigDict(extra="forbid") - url: str | None = None + url: str = Field(min_length=1) class PluginCatalogEntry(BaseModel): @@ -113,12 +113,12 @@ class PluginCatalogEntry(BaseModel): name: str plugin_type: PluginType - description: str = "" + description: str = Field(min_length=1) package: PluginPackageInfo install: PluginInstallInfo entry_point: PluginEntryPointInfo - compatibility: PluginCompatibility | None = None - docs: PluginDocsInfo | None = None + compatibility: PluginCompatibility + docs: PluginDocsInfo class PluginCatalogRuntimePlugin(BaseModel): @@ -137,11 +137,11 @@ class PluginCatalogPackage(BaseModel): model_config = ConfigDict(extra="forbid") name: str - description: str = "" + description: str = Field(min_length=1) install: PluginInstallInfo - compatibility: PluginCompatibility | None = None - docs: PluginDocsInfo | None = None - plugins: list[PluginCatalogRuntimePlugin] = Field(default_factory=list) + compatibility: PluginCompatibility + docs: PluginDocsInfo + plugins: list[PluginCatalogRuntimePlugin] = Field(min_length=1) def entries(self) -> list[PluginCatalogEntry]: """Flatten nested runtime plugins while preserving package-level metadata.""" @@ -427,6 +427,8 @@ def _required_catalog_object( def _validate_catalog_object_keys(context: str, value: dict[str, object], expected_keys: set[str]) -> None: keys = set(value) if keys != expected_keys: + # Catalog v2 is strict by design: additive wire-schema changes should bump + # schema_version so older CLIs do not silently ignore trust-sensitive fields. raise PluginCatalogError( f"{context} has invalid fields; expected {{{_format_catalog_keys(expected_keys)}}}, " f"got {{{_format_catalog_keys(keys)}}}" diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py index 4134b315b..4d3a230e7 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py @@ -5,6 +5,7 @@ import hashlib import json +import os from datetime import datetime, timezone from pathlib import Path from urllib.error import HTTPError, URLError @@ -182,8 +183,13 @@ def _save_catalog_cache(self, catalog: PluginCatalogConfig, catalog_payload: dic "fetched_at": datetime.now(timezone.utc).isoformat(), "catalog": catalog_payload, } - with open(self._cache_file(catalog), "w") as f: - json.dump(cache_payload, f, indent=2, sort_keys=True) + cache_file = self._cache_file(catalog) + temp_path = cache_file.with_name(f"{cache_file.name}.{os.getpid()}.tmp") + try: + temp_path.write_text(json.dumps(cache_payload, indent=2, sort_keys=True), encoding="utf-8") + temp_path.replace(cache_file) + finally: + temp_path.unlink(missing_ok=True) def _cache_file(self, catalog: PluginCatalogConfig) -> Path: url_hash = hashlib.sha256(catalog.url.encode("utf-8")).hexdigest()[:12] @@ -312,6 +318,8 @@ def _fetch_remote_catalog(url: str) -> dict[str, object]: status = getattr(response, "status", 200) if isinstance(status, int) and status >= 400: raise PluginCatalogError(f"Failed to fetch plugin catalog {url!r}: HTTP {status}") + # Read one byte past the limit so oversized chunked responses are + # rejected without keeping the full response body in memory. content = response.read(MAX_PLUGIN_CATALOG_SIZE_BYTES + 1) except HTTPError as e: raise PluginCatalogError(f"Failed to fetch plugin catalog {url!r}: HTTP {e.code}") from e diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index 088326859..ca0e1b134 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -15,7 +15,6 @@ from data_designer.cli.plugin_catalog import ( DATA_DESIGNER_PLUGIN_PACKAGE_PREFIX, - DEFAULT_PLUGIN_CATALOG_ALIAS, PLUGIN_ENTRY_POINT_GROUP, CompatibilityResult, InstalledPluginInfo, @@ -77,31 +76,6 @@ def search_entries( if all(token in _entry_search_text(entry) for token in query_tokens) ] - def get_entry( - self, - name: str, - catalog_alias: str | None = None, - *, - refresh: bool = False, - include_incompatible: bool = True, - ) -> PluginCatalogEntry: - """Return a catalog entry by runtime plugin name or package name.""" - entries = self.list_entries(catalog_alias, refresh=refresh, include_incompatible=True) - canonical_name = canonicalize_name(name) - runtime_matches = [entry for entry in entries if entry.name == name] - package_matches = [entry for entry in entries if canonicalize_name(entry.package.name) == canonical_name] - matches = runtime_matches or package_matches - compatible_matches = [entry for entry in matches if self.evaluate_compatibility(entry).is_compatible] - if compatible_matches: - return sorted(compatible_matches, key=lambda entry: (canonicalize_name(entry.package.name), entry.name))[0] - if matches and include_incompatible: - return sorted(matches, key=lambda entry: (canonicalize_name(entry.package.name), entry.name))[0] - - resolved_alias = catalog_alias or DEFAULT_PLUGIN_CATALOG_ALIAS - if matches: - raise ValueError(f"Plugin package {name!r} was found in catalog {resolved_alias!r}, but is not compatible") - raise ValueError(f"Plugin or package {name!r} was not found in catalog {resolved_alias!r}") - def get_package_entries( self, package: str, @@ -138,9 +112,6 @@ def group_entries_by_package(entries: Iterable[PluginCatalogEntry]) -> dict[str, def evaluate_compatibility(self, entry: PluginCatalogEntry) -> CompatibilityResult: """Evaluate whether a catalog entry is compatible with the local environment.""" compatibility = entry.compatibility - if compatibility is None: - return CompatibilityResult(is_compatible=True, reasons=[]) - reasons = [] reasons.extend( self._evaluate_target( @@ -203,14 +174,11 @@ def list_installed_plugins(self) -> list[InstalledPluginInfo]: def _evaluate_target( self, *, - target: PluginCompatibilityTarget | None, + target: PluginCompatibilityTarget, label: str, version: str | None, marker_environment: dict[str, str], ) -> list[str]: - if target is None or not target.specifier: - return [] - marker_error = _marker_error(target.marker, marker_environment) if marker_error is not None: return [f"{label} marker {target.marker!r} is invalid: {marker_error}"] diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 8cfdc980b..d22a15c48 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -170,7 +170,6 @@ def test_run_info_renders_package_metadata_with_nested_runtime_plugins( assert all("install" not in plugin for plugin in metadata["plugins"]) assert all("compatibility" not in plugin for plugin in metadata["plugins"]) assert all("docs" not in plugin for plugin in metadata["plugins"]) - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "text-transform", "local", @@ -220,7 +219,6 @@ def test_run_info_rejects_runtime_plugin_name_that_is_not_package_alias( controller.run_info("text-column", catalog_alias="local") assert exc_info.value.exit_code == 1 - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "text-column", "local", @@ -239,7 +237,7 @@ def test_run_install_dry_run_renders_plan_without_installing( ) -> None: entry = _entry() catalog = _catalog(trusted=True) - plan = _plan(catalog, data_designer_protection="pinned to installed data-designer 0.5.10") + plan = _plan(catalog, data_designer_protection="pinned installed Data Designer packages; data-designer 0.5.10") controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) @@ -247,7 +245,6 @@ def test_run_install_dry_run_renders_plan_without_installing( controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "data-designer-text-transform", "local", @@ -257,7 +254,9 @@ def test_run_install_dry_run_renders_plan_without_installing( controller.install_service.install.assert_not_called() controller.install_service.verify_entry_points.assert_not_called() mock_print_info.assert_any_call("Dry run complete; no changes made") - mock_console.print.assert_any_call(" Data Designer: [bold]pinned to installed data-designer 0.5.10[/bold]") + mock_console.print.assert_any_call( + " Data Designer: [bold]pinned installed Data Designer packages; data-designer 0.5.10[/bold]" + ) assert all("Runtime plugins" not in str(call_args.args[0]) for call_args in mock_console.print.call_args_list) assert mock_console.print.call_count >= 1 @@ -282,7 +281,6 @@ def test_run_install_blocks_incompatible_package( controller.run_install("data-designer-text-transform", catalog_alias="local") assert exc_info.value.exit_code == 1 - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "data-designer-text-transform", "local", @@ -309,7 +307,6 @@ def test_run_install_rejects_runtime_plugin_name_as_target( controller.run_install("text-column", catalog_alias="local") assert exc_info.value.exit_code == 1 - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "text-column", "local", @@ -371,7 +368,6 @@ def test_run_install_dry_run_allows_incompatible_entry_for_inspection( controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "data-designer-text-transform", "local", @@ -500,7 +496,6 @@ def test_run_uninstall_dry_run_renders_plan_without_uninstalling( controller.run_uninstall("data-designer-text-transform", catalog_alias="local", dry_run=True) - controller.catalog_service.get_entry.assert_not_called() controller.catalog_service.get_package_entries.assert_called_once_with( "data-designer-text-transform", "local", diff --git a/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-invalid-install.json b/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-invalid-install.json new file mode 100644 index 000000000..4c6065a3f --- /dev/null +++ b/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-invalid-install.json @@ -0,0 +1,36 @@ +{ + "schema_version": 2, + "packages": [ + { + "name": "data-designer-invalid-install", + "description": "Invalid install fixture", + "install": { + "requirement": "other-package==0.1.0" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-invalid-install/" + }, + "plugins": [ + { + "name": "invalid-install", + "plugin_type": "column-generator", + "entry_point": { + "group": "data_designer.plugins", + "name": "invalid-install", + "value": "data_designer_invalid_install.plugin:plugin" + } + } + ] + } + ] +} diff --git a/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-unsupported-version.json b/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-unsupported-version.json new file mode 100644 index 000000000..8603f09a8 --- /dev/null +++ b/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-unsupported-version.json @@ -0,0 +1,4 @@ +{ + "schema_version": 999, + "packages": [] +} diff --git a/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-valid.json b/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-valid.json new file mode 100644 index 000000000..e295ccffb --- /dev/null +++ b/packages/data-designer/tests/cli/fixtures/upstream-catalogs/catalog-valid.json @@ -0,0 +1,204 @@ +{ + "schema_version": 2, + "packages": [ + { + "name": "data-designer-compatible-column", + "description": "Compatible index-backed column generator fixture", + "install": { + "requirement": "data-designer-compatible-column", + "index_url": "https://docs.example.test/simple/" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-compatible-column/" + }, + "plugins": [ + { + "name": "compatible-column", + "plugin_type": "column-generator", + "entry_point": { + "group": "data_designer.plugins", + "name": "compatible-column", + "value": "data_designer_compatible_column.plugin:plugin" + } + } + ] + }, + { + "name": "data-designer-git-seed-reader", + "description": "Compatible Git direct reference seed reader fixture", + "install": { + "requirement": "data-designer-git-seed-reader @ git+https://github.com/NVIDIA-NeMo/DataDesignerPlugins.git@data-designer-git-seed-reader/v0.2.0#subdirectory=plugins/data-designer-git-seed-reader" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-git-seed-reader/" + }, + "plugins": [ + { + "name": "compatible-git-seed-reader", + "plugin_type": "seed-reader", + "entry_point": { + "group": "data_designer.plugins", + "name": "compatible-git-seed-reader", + "value": "data_designer_git_seed_reader.plugin:plugin" + } + } + ] + }, + { + "name": "data-designer-url-processor", + "description": "Compatible direct URL processor fixture", + "install": { + "requirement": "data-designer-url-processor @ https://packages.example.test/data_designer_url_processor-0.2.1-py3-none-any.whl" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-url-processor/" + }, + "plugins": [ + { + "name": "compatible-url-processor", + "plugin_type": "processor", + "entry_point": { + "group": "data_designer.plugins", + "name": "compatible-url-processor", + "value": "data_designer_url_processor.plugin:plugin" + } + } + ] + }, + { + "name": "data-designer-python312-column", + "description": "Python compatibility rejection fixture", + "install": { + "requirement": "data-designer-python312-column", + "index_url": "https://docs.example.test/simple/" + }, + "compatibility": { + "python": { + "specifier": ">=3.12" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-python312-column/" + }, + "plugins": [ + { + "name": "python312-column", + "plugin_type": "column-generator", + "entry_point": { + "group": "data_designer.plugins", + "name": "python312-column", + "value": "data_designer_python312_column.plugin:plugin" + } + } + ] + }, + { + "name": "data-designer-future-dd-processor", + "description": "Data Designer compatibility rejection fixture", + "install": { + "requirement": "data-designer-future-dd-processor", + "index_url": "https://docs.example.test/simple/" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=999.0", + "specifier": ">=999.0", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-future-dd-processor/" + }, + "plugins": [ + { + "name": "future-dd-processor", + "plugin_type": "processor", + "entry_point": { + "group": "data_designer.plugins", + "name": "future-dd-processor", + "value": "data_designer_future_dd_processor.plugin:plugin" + } + } + ] + }, + { + "name": "data-designer-multi-plugin-package", + "description": "Multi-plugin package fixture", + "install": { + "requirement": "data-designer-multi-plugin-package", + "index_url": "https://docs.example.test/simple/" + }, + "compatibility": { + "python": { + "specifier": ">=3.10" + }, + "data_designer": { + "requirement": "data-designer>=0.5.7", + "specifier": ">=0.5.7", + "marker": null + } + }, + "docs": { + "url": "https://docs.example.test/plugins/data-designer-multi-plugin-package/" + }, + "plugins": [ + { + "name": "multi-seed-reader", + "plugin_type": "seed-reader", + "entry_point": { + "group": "data_designer.plugins", + "name": "multi-seed-reader", + "value": "data_designer_multi_plugin_package.seed:plugin" + } + }, + { + "name": "multi-processor", + "plugin_type": "processor", + "entry_point": { + "group": "data_designer.plugins", + "name": "multi-processor", + "value": "data_designer_multi_plugin_package.processor:plugin" + } + } + ] + } + ] +} diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py index d02c222d5..e437daf0b 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py @@ -4,20 +4,25 @@ from __future__ import annotations import json +import os from pathlib import Path from unittest.mock import Mock, patch from urllib.error import HTTPError import pytest +from pydantic import ValidationError from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_CATALOG_ALIAS, DEFAULT_PLUGIN_CATALOG_URL_ENV_VAR, MAX_PLUGIN_CATALOG_SIZE_BYTES, + PluginCatalog, PluginCatalogError, ) from data_designer.cli.repositories.plugin_catalog_repository import PluginCatalogRepository, normalize_catalog_location +UPSTREAM_CATALOG_FIXTURES_DIR = Path(__file__).parents[1] / "fixtures" / "upstream-catalogs" + def test_repository_includes_default_nvidia_catalog(tmp_path: Path) -> None: repository = PluginCatalogRepository(tmp_path) @@ -275,6 +280,63 @@ def test_load_catalog_accepts_schema_v2_package_catalog(tmp_path: Path) -> None: assert catalog.plugins[0].install.index_url == "https://docs.example.test/simple/" +def test_consumer_accepts_upstream_valid_catalog_fixture(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("upstream", str(UPSTREAM_CATALOG_FIXTURES_DIR / "catalog-valid.json")) + + catalog = repository.load_catalog("upstream", refresh=True) + + assert [package.name for package in catalog.packages] == [ + "data-designer-compatible-column", + "data-designer-git-seed-reader", + "data-designer-url-processor", + "data-designer-python312-column", + "data-designer-future-dd-processor", + "data-designer-multi-plugin-package", + ] + assert [entry.name for entry in catalog.plugins][-2:] == ["multi-seed-reader", "multi-processor"] + + +def test_consumer_rejects_upstream_invalid_install_fixture(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("upstream", str(UPSTREAM_CATALOG_FIXTURES_DIR / "catalog-invalid-install.json")) + + with pytest.raises(PluginCatalogError, match="expected a requirement for 'data-designer-invalid-install'"): + repository.load_catalog("upstream", refresh=True) + + +def test_consumer_rejects_upstream_unsupported_version_fixture(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("upstream", str(UPSTREAM_CATALOG_FIXTURES_DIR / "catalog-unsupported-version.json")) + + with pytest.raises(PluginCatalogError, match="unsupported catalog schema_version 999"): + repository.load_catalog("upstream", refresh=True) + + +def test_catalog_model_requires_contract_required_package_metadata() -> None: + package = _package_entry() + package.pop("compatibility") + + with pytest.raises(ValidationError, match="compatibility"): + PluginCatalog.model_validate(_catalog_payload(packages=[package])) + + +def test_catalog_model_requires_non_empty_runtime_plugins() -> None: + package = _package_entry(plugins=[]) + + with pytest.raises(ValidationError, match="plugins"): + PluginCatalog.model_validate(_catalog_payload(packages=[package])) + + +def test_fetches_production_catalog_when_enabled(tmp_path: Path) -> None: + if os.getenv("DATA_DESIGNER_TEST_REMOTE_PLUGIN_CATALOG") != "1": + pytest.skip("Set DATA_DESIGNER_TEST_REMOTE_PLUGIN_CATALOG=1 to run the live catalog smoke test") + + catalog = PluginCatalogRepository(tmp_path).load_catalog(refresh=True) + + assert catalog.packages + + def test_load_catalog_accepts_equivalent_data_designer_marker_quoting(tmp_path: Path) -> None: package = _package_entry() package["compatibility"]["data_designer"] = { diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index 7b8b11e61..75952bf24 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -51,7 +51,7 @@ def test_search_entries_matches_name_type_package_and_docs(tmp_path: Path) -> No def test_evaluate_compatibility_reports_data_designer_constraint(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - entry = service.get_entry("future-plugin", "local") + entry = _entry_by_name(service.list_entries("local", include_incompatible=True), "future-plugin") result = service.evaluate_compatibility(entry) @@ -140,7 +140,7 @@ def test_evaluate_compatibility_accepts_local_dev_version_above_lower_bound(tmp_ python_version="3.11.0", data_designer_version="0.5.10.dev18+604fdd96", ) - entry = service.get_entry("compatible-plugin", "local") + entry = _entry_by_name(service.list_entries("local", include_incompatible=True), "compatible-plugin") result = service.evaluate_compatibility(entry) @@ -148,66 +148,6 @@ def test_evaluate_compatibility_accepts_local_dev_version_above_lower_bound(tmp_ assert result.reasons == [] -def test_get_entry_rejects_incompatible_plugin_when_requested(tmp_path: Path) -> None: - repository = _repository_with_catalog(tmp_path) - service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - - with pytest.raises(ValueError, match="not compatible"): - service.get_entry("future-plugin", "local", include_incompatible=False) - - -def test_get_entry_resolves_package_name() -> None: - repository = Mock(spec=PluginCatalogRepository) - repository.load_catalog.return_value = PluginCatalog.model_validate( - { - "schema_version": 2, - "packages": [ - _package( - package_name="data-designer-package-target", - data_designer_specifier=">=0.5.7", - plugins=[ - _runtime_plugin(name="package-column", plugin_type="column-generator"), - _runtime_plugin(name="package-processor", plugin_type="processor"), - ], - ), - ], - } - ) - service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - - entry = service.get_entry("data-designer-package-target", "local", include_incompatible=True) - - assert entry.name == "package-column" - assert entry.package.name == "data-designer-package-target" - - -def test_get_entry_prefers_runtime_plugin_name_over_package_name() -> None: - repository = Mock(spec=PluginCatalogRepository) - repository.load_catalog.return_value = PluginCatalog.model_validate( - { - "schema_version": 2, - "packages": [ - _package( - package_name="alpha", - data_designer_specifier=">=0.5.7", - plugins=[_runtime_plugin(name="package-plugin", plugin_type="processor")], - ), - _package( - package_name="data-designer-runtime-source", - data_designer_specifier=">=0.5.7", - plugins=[_runtime_plugin(name="alpha", plugin_type="processor")], - ), - ], - } - ) - service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") - - entry = service.get_entry("alpha", "local", include_incompatible=True) - - assert entry.name == "alpha" - assert entry.package.name == "data-designer-runtime-source" - - def test_get_package_entries_resolves_package_alias() -> None: repository = Mock(spec=PluginCatalogRepository) repository.load_catalog.return_value = PluginCatalog.model_validate( @@ -353,6 +293,10 @@ def _repository_with_catalog(tmp_path: Path) -> PluginCatalogRepository: return repository +def _entry_by_name(entries: list[PluginCatalogEntry], name: str) -> PluginCatalogEntry: + return next(entry for entry in entries if entry.name == name) + + def _catalog_payload() -> dict: return { "schema_version": 2, From d8e1d6acd63d0e4deb9b596cb35511da0abe9131 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 01:09:08 +0000 Subject: [PATCH 27/34] fix no-args help exit code --- .../src/data_designer/cli/lazy_group.py | 6 ++++++ packages/data-designer/tests/cli/test_main.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/packages/data-designer/src/data_designer/cli/lazy_group.py b/packages/data-designer/src/data_designer/cli/lazy_group.py index f498b3230..6b6f055aa 100644 --- a/packages/data-designer/src/data_designer/cli/lazy_group.py +++ b/packages/data-designer/src/data_designer/cli/lazy_group.py @@ -83,6 +83,12 @@ def create_lazy_typer_group( """ class LazyTyperGroup(TyperGroup): + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + if not args and self.no_args_is_help and not ctx.resilient_parsing: + click.echo(ctx.get_help(), color=ctx.color) + ctx.exit(0) + return super().parse_args(ctx, args) + def list_commands(self, ctx: click.Context) -> list[str]: eager = super().list_commands(ctx) lazy_names = [name for name in lazy_subcommands if name not in eager] diff --git a/packages/data-designer/tests/cli/test_main.py b/packages/data-designer/tests/cli/test_main.py index db805adb7..2e155147f 100644 --- a/packages/data-designer/tests/cli/test_main.py +++ b/packages/data-designer/tests/cli/test_main.py @@ -225,6 +225,22 @@ def test_app_help_keeps_config_and_plugin_commands_reachable() -> None: assert "catalog" in plugin_result.output +def test_no_args_help_exits_successfully_for_lazy_groups() -> None: + root_result = runner.invoke(app, []) + plugin_result = runner.invoke(app, ["plugin"]) + plugin_catalog_result = runner.invoke(app, ["plugin", "catalog"]) + + assert root_result.exit_code == 0 + assert "Data Designer CLI" in root_result.output + assert "plugin" in root_result.output + assert plugin_result.exit_code == 0 + assert "Discover, install, and uninstall" in plugin_result.output + assert "catalog" in plugin_result.output + assert plugin_catalog_result.exit_code == 0 + assert "Manage plugin catalog aliases" in plugin_catalog_result.output + assert "add" in plugin_catalog_result.output + + def test_app_does_not_expose_legacy_plugins_command() -> None: result = runner.invoke(app, ["plugins", "--help"]) From 13a1c9b596406460461415ac1786b003a200e77e Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 01:15:06 +0000 Subject: [PATCH 28/34] make plugin docs links robust --- .../controllers/plugin_catalog_controller.py | 10 +++++++++- .../test_plugin_catalog_controller.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index bf65c71a0..df518c363 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -8,7 +8,9 @@ import typer from pydantic import ValidationError +from rich.style import Style from rich.table import Table +from rich.text import Text from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_CATALOG_ALIAS, @@ -490,7 +492,7 @@ def _display_catalog_entries(self, entries: list[PluginCatalogEntry]) -> None: entry.description, _format_runtime_plugins(package_entries), "yes" if compatibility.is_compatible else "no", - docs_url, + _format_docs_link(docs_url), ) console.print(table) @@ -538,6 +540,12 @@ def _format_runtime_plugins(entries: list[PluginCatalogEntry]) -> str: return ", ".join(f"{entry.name} ({entry.plugin_type.value})" for entry in entries) +def _format_docs_link(docs_url: str | None) -> Text: + if not docs_url: + return Text("") + return Text("docs", style=Style(color=NordColor.NORD7.value, link=docs_url)) + + def _runtime_plugin_metadata(entry: PluginCatalogEntry) -> dict[str, object]: return { "name": entry.name, diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index d22a15c48..ba44a68c0 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -3,12 +3,15 @@ from __future__ import annotations +from io import StringIO from pathlib import Path from unittest.mock import MagicMock, call, patch import pytest import typer +from rich.console import Console from rich.table import Table +from rich.text import Text from data_designer.cli.controllers.plugin_catalog_controller import PluginCatalogController from data_designer.cli.plugin_catalog import ( @@ -115,6 +118,22 @@ def test_run_list_renders_package_first_catalog_table( "Docs", ] assert list(printed_tables[0].columns[1].cells) == ["Transform text records"] + docs_cell = list(printed_tables[0].columns[4].cells)[0] + assert isinstance(docs_cell, Text) + assert docs_cell.plain == "docs" + assert docs_cell.style is not None + assert docs_cell.style.link == "https://docs.example.test/plugins/data-designer-text-transform/" + + rendered_output = StringIO() + narrow_console = Console( + file=rendered_output, + force_terminal=True, + color_system="standard", + width=60, + legacy_windows=False, + ) + narrow_console.print(printed_tables[0]) + assert "https://docs.example.test/plugins/data-designer-text-transform/" in rendered_output.getvalue() controller.catalog_service.group_entries_by_package.assert_called_once_with(package_entries) From 3d0e5265939d1010927fcabb63d49819376e2d77 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 01:44:40 +0000 Subject: [PATCH 29/34] document plugin CLI catalog workflows --- architecture/cli.md | 24 +++--- .../src/data_designer/cli/README.md | 78 +++++++++++++++---- 2 files changed, 76 insertions(+), 26 deletions(-) diff --git a/architecture/cli.md b/architecture/cli.md index 9ed703626..9bfba6377 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -1,6 +1,6 @@ # CLI -The CLI (`data-designer`) provides an interactive command-line interface for configuring models, providers, tools, and personas, discovering/installing plugin packages from catalogs, and running dataset generation. It uses a layered architecture for setup workflows and delegates generation to the public `DataDesigner` API. +The CLI (`data-designer`) provides an interactive command-line interface for configuring models, providers, MCP providers, and tools, downloading managed persona datasets, discovering, installing, and uninstalling plugin packages from catalogs, and running dataset generation. It uses a layered architecture for setup workflows and delegates generation to the public `DataDesigner` API. Source: `packages/data-designer/src/data_designer/cli/` @@ -22,7 +22,7 @@ The CLI is built on Typer with lazy command loading to keep startup fast. Config ### Layering Pattern (Setup Workflows) -Config management commands (models, providers, tools, personas) follow a consistent four-layer pattern: +Config management commands (models, providers, MCP providers, tools) follow a consistent four-layer pattern: | Layer | Role | Example | |-------|------|---------| @@ -31,7 +31,8 @@ Config management commands (models, providers, tools, personas) follow a consist | **Service** | Domain rules: uniqueness, merge, delete-all | `ModelService.add/update/delete` over `ModelRepository` | | **Repository** | File I/O for typed config registries | `ModelRepository` extends `ConfigRepository[ModelConfigRegistry]` | -Repositories: `ModelRepository`, `ProviderRepository`, `ToolRepository`, `MCPProviderRepository`, `PersonaRepository`. +Repositories: `ModelRepository`, `ProviderRepository`, `MCPProviderRepository`, and `ToolRepository`. +`PersonaRepository` provides read-only locale metadata for managed persona dataset downloads. Services mirror the repository domains with business logic (validation, conflict resolution). @@ -39,9 +40,9 @@ Plugin catalog commands use the same layering shape: | Layer | Role | Example | |-------|------|---------| -| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugin list/search/info/install/uninstall/installed/catalog` → `PluginCatalogController(DATA_DESIGNER_HOME)` | +| **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugin` subcommands (`list`, `search`, `info`, `install`, `uninstall`, `installed`, `catalog`) → `PluginCatalogController(DATA_DESIGNER_HOME)` | | **Controller** | UX flow: catalog tables, package metadata, compatibility display, package mutation confirmations | `PluginCatalogController` composes catalog + install services | -| **Service** | Domain rules: package-first flattening, compatibility checks, install/uninstall planning, entry point verification | `PluginCatalogService`, `PluginInstallService` | +| **Service** | Domain rules: package-first flattening, compatibility checks, install and uninstall planning, runtime entry-point verification | `PluginCatalogService`, `PluginInstallService` | | **Repository** | File/cache I/O for catalog aliases and catalog documents | `PluginCatalogRepository` | The built-in `nvidia` catalog points at `https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`. `NVIDIA-NeMo/DataDesignerPlugins` defines the package-first catalog shape: top-level packages carry install metadata, compatibility constraints, docs, and nested runtime plugins. The CLI flattens nested plugins for list/search display, but `info`, `install`, and `uninstall` resolve package names or package aliases so environment mutations target the package distribution. Package aliases come from the `data-designer-{alias}` package-name pattern; for example, `data-designer-calculator` can be addressed as `calculator`. @@ -90,9 +91,10 @@ User invokes command (e.g., `data-designer plugin install calculator`) → PluginInstallService builds a pip/uv install plan for the package requirement. In active uv projects it uses `uv add` so the package lands in `pyproject.toml`; otherwise it mutates the active Python environment with `uv pip install` or pip. - The active `data-designer` distribution is skipped, excluded, or pinned from - replacement depending on package-manager capabilities. - → PluginInstallService verifies declared package entry points after installation + The active Data Designer package family (`data-designer`, + `data-designer-config`, and `data-designer-engine`) is skipped, excluded, or + pinned from replacement depending on package-manager capabilities. + → PluginInstallService verifies declared runtime entry points after installation ``` ``` @@ -101,7 +103,7 @@ User invokes command (e.g., `data-designer plugin uninstall calculator`) → PluginInstallService builds a pip/uv uninstall plan for the package distribution. Active uv projects remove the dependency from project metadata without a uv sync, then uninstall the package from the active environment. - → PluginInstallService verifies declared package entry points are no longer discovered + → PluginInstallService verifies declared runtime entry points are no longer discovered ``` ### Generation @@ -116,8 +118,8 @@ User invokes command (e.g., `data-designer create config.yaml`) - **Lazy command loading** keeps `data-designer --help` responsive: command modules (and their heavy dependencies, such as the engine and model stacks) load only when a command is invoked, not at process startup. - **Controller/service/repo for setup workflows, direct API for generation** — config and plugin catalog workflows benefit from the layered pattern (testable services, swappable repositories). Generation doesn't need this indirection; it delegates to the same `DataDesigner` class that Python users call directly. -- **`DATA_DESIGNER_HOME`** centralizes all CLI-managed state (model configs, provider configs, tool configs, personas) in a single directory, defaulting to `~/.data_designer/`. -- **Package-first plugin catalogs** keep install metadata at the package boundary while allowing one package to expose multiple runtime plugins through entry points. +- **`DATA_DESIGNER_HOME`** centralizes CLI-managed state (model configs, provider configs, MCP provider configs, tool configs, managed assets, plugin catalog aliases, and catalog caches) in a single directory, defaulting to `~/.data-designer/`. +- **Package-first plugin catalogs** keep install metadata at the package boundary while allowing one package to expose multiple runtime plugins through runtime entry points. - **Rich-based UI** provides formatted tables, progress bars, and interactive prompts without requiring a web interface. ## Cross-References diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 9899cb9ed..d865810e4 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -1,16 +1,19 @@ # 🎨 NeMo Data Designer CLI -This directory contains the Command-Line Interface (CLI) for configuring model providers, model configurations, managed assets, and plugin catalogs used in Data Designer. +This directory contains the Command-Line Interface (CLI) for configuring model providers, model configurations, MCP providers, tool configs, managed assets, and plugin catalogs used in Data Designer. ## Overview The CLI provides an interactive interface for managing: - **Model Providers**: LLM API endpoints (NVIDIA, OpenAI, Anthropic, custom providers) - **Model Configs**: Specific model configurations with inference parameters +- **MCP Providers**: MCP server configurations for tool integration +- **Tool Configs**: Tool definitions used by configured models and workflows +- **Managed Assets**: Persona dataset downloads under the Data Designer home directory - **Plugin Catalogs**: Catalog aliases for discovering Data Designer plugin packages -- **Plugin Installs**: Safe install-plan rendering, package-manager execution with active Data Designer protection, and entry point verification +- **Plugin Installs**: Safe install-plan rendering, package-manager execution with active Data Designer package-family protection, and runtime entry-point verification -Configuration files are stored in `~/.data-designer/` by default and can be referenced by Data Designer workflows. +Configuration files and CLI-managed state are stored in `~/.data-designer/` by default. ## Architecture @@ -19,7 +22,7 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis ``` ┌─────────────────────────────────────────────────────────────┐ │ Commands │ -│ Entry points for CLI commands (list, providers, plugin) │ +│ Entry points for CLI commands (config, download, plugin) │ └─────────────────────────────────────────────────────────────┘ │ ▼ @@ -52,10 +55,13 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - Handle top-level error reporting - **Files**: - `list.py`: List current configurations + - `mcp.py`: Configure MCP providers - `models.py`: Configure models - `providers.py`: Configure providers + - `download.py`: Download managed assets - `plugin.py`: Discover, install, and uninstall plugin packages from catalogs - `reset.py`: Reset/delete configurations + - `tools.py`: Configure tool configs #### 2. **Controllers** (`controllers/`) - **Purpose**: Orchestrate user workflows and coordinate between services, forms, and UI @@ -65,9 +71,12 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - Handle user navigation and session state - Manage associated resource deletion (e.g., deleting models when provider is deleted) - **Files**: + - `download_controller.py`: Orchestrates managed asset download workflows + - `mcp_provider_controller.py`: Orchestrates MCP provider configuration workflows - `model_controller.py`: Orchestrates model configuration workflows - `provider_controller.py`: Orchestrates provider configuration workflows - `plugin_catalog_controller.py`: Orchestrates plugin catalog browsing, alias management, and package workflows + - `tool_controller.py`: Orchestrates tool configuration workflows **Key Features**: - **Associated Resource Management**: When deleting a provider, the controller checks for associated models and prompts the user to delete them together @@ -81,10 +90,12 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - Coordinate between multiple repositories when needed - Handle default management (e.g., default provider selection) - **Files**: + - `mcp_provider_service.py`: MCP provider configuration business logic - `model_service.py`: Model configuration business logic - `provider_service.py`: Provider business logic - - `plugin_catalog_service.py`: Plugin catalog discovery, search, compatibility checks, and installed entry point listing - - `plugin_install_service.py`: Plugin install/uninstall plan resolution, package-manager execution, active Data Designer protection, and runtime verification + - `plugin_catalog_service.py`: Plugin catalog discovery, search, compatibility checks, and installed runtime entry-point listing + - `plugin_install_service.py`: Plugin install and uninstall plan resolution, package-manager execution, active Data Designer package-family protection, and runtime entry-point verification + - `tool_service.py`: Tool configuration business logic **Key Methods**: - `list_all()`: Get all configured items @@ -97,17 +108,20 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `set_default()`, `get_default()`: Manage default provider (providers only) #### 4. **Repositories** (`repositories/`) -- **Purpose**: Handle data persistence (YAML file I/O) +- **Purpose**: Handle data persistence and read-only reference metadata - **Responsibilities**: - Load configuration from YAML files - Save configuration to YAML files - - Check file existence - - Delete configuration files + - Check file existence and delete configuration files where applicable + - Provide read-only metadata for built-in managed assets - **Files**: - `base.py`: Abstract base repository with common operations + - `mcp_provider_repository.py`: MCP provider configuration persistence - `model_repository.py`: Model configuration persistence + - `persona_repository.py`: Read-only persona locale metadata - `provider_repository.py`: Provider persistence - `plugin_catalog_repository.py`: Plugin catalog aliases, catalog fetching, and URL-keyed catalog cache + - `tool_repository.py`: Tool configuration persistence **Base Repository Pattern**: ```python @@ -129,8 +143,10 @@ class ConfigRepository(ABC, Generic[T]): - `builder.py`: Abstract form builder base - `field.py`: Form field types (TextField, SelectField, NumericField) - `form.py`: Form container and prompt orchestration + - `mcp_provider_builder.py`: Interactive MCP provider configuration builder - `model_builder.py`: Interactive model configuration builder - `provider_builder.py`: Interactive provider configuration builder + - `tool_builder.py`: Interactive tool configuration builder **Form Features**: - Field-level validation @@ -159,7 +175,7 @@ class ConfigRepository(ABC, Generic[T]): ## Configuration Files -The CLI manages YAML configuration files and plugin catalog caches under `~/.data-designer/`: +The CLI manages YAML configuration files, managed assets, and plugin catalog caches under `~/.data-designer/`: ### `~/.data-designer/model_providers.yaml` @@ -213,6 +229,38 @@ model_configs: max_parallel_requests: 4 ``` +### `~/.data-designer/mcp_providers.yaml` + +Stores MCP provider configurations: + +```yaml +providers: + - name: local-tools + provider_type: stdio + command: python + args: + - "-m" + - my_mcp_server +``` + +### `~/.data-designer/tool_configs.yaml` + +Stores tool configurations that reference MCP providers: + +```yaml +tool_configs: + - tool_alias: research-tools + providers: + - local-tools + max_tool_call_turns: 5 +``` + +### `~/.data-designer/managed-assets/` + +Stores managed assets downloaded by CLI commands such as +`data-designer download personas`. Set `DATA_DESIGNER_MANAGED_ASSETS_PATH` to +store managed assets outside `DATA_DESIGNER_HOME`. + ### `~/.data-designer/plugin_catalogs.yaml` Stores user-added plugin catalog aliases. The built-in NVIDIA catalog points at @@ -290,19 +338,19 @@ data-designer plugin list # Search a specific catalog data-designer plugin --catalog research search transform -# Show metadata, compatibility, docs, and exact install command +# Show metadata, compatibility, docs, and the install plan data-designer plugin info github -# Install a plugin package from a catalog and verify package registration +# Install a plugin package from a catalog and verify declared runtime entry points data-designer plugin install github --yes -# Preview the install command without mutating the environment +# Preview the install plan without mutating the environment data-designer plugin install github --dry-run -# Uninstall a plugin package from a catalog and verify package registration is removed +# Uninstall a plugin package from a catalog and verify declared runtime entry points are removed data-designer plugin uninstall github --yes -# Preview the uninstall command without mutating the environment +# Preview the uninstall plan without mutating the environment data-designer plugin uninstall github --dry-run # Add and manage catalog aliases From 1a9f46852d723f2d97f6573fbc2b0e6d94c82fe1 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 01:44:49 +0000 Subject: [PATCH 30/34] clarify plugin entry point verification --- .../src/data_designer/cli/commands/plugin.py | 14 ++++++++++---- .../cli/controllers/plugin_catalog_controller.py | 8 ++++---- .../data-designer/src/data_designer/cli/main.py | 6 +++--- .../src/data_designer/cli/plugin_catalog.py | 4 ++-- .../cli/services/plugin_catalog_service.py | 2 +- .../cli/services/plugin_install_service.py | 8 ++++---- .../controllers/test_plugin_catalog_controller.py | 10 ++++++---- 7 files changed, 30 insertions(+), 22 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/commands/plugin.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py index e574ea0dd..eaf5940b7 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugin.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -115,7 +115,10 @@ def install_command( "auto", "--manager", click_type=click.Choice(["auto", "uv", "pip"]), - help="Package manager to use. uv adds to the active project when one is detected; pip mutates the environment.", + help=( + "Package manager to use. auto prefers uv; uv adds to the active project when one is detected; " + "pip mutates the environment." + ), ), yes: bool = typer.Option( False, @@ -129,7 +132,7 @@ def install_command( help="Print the install plan without mutating the current environment.", ), ) -> None: - """Install one Data Designer plugin package, then verify package registration.""" + """Install one Data Designer plugin package, then verify declared runtime entry points.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_install( package, @@ -161,7 +164,10 @@ def uninstall_command( "auto", "--manager", click_type=click.Choice(["auto", "uv", "pip"]), - help="Package manager to use. uv removes from the active project when one is detected; pip mutates the environment.", + help=( + "Package manager to use. auto prefers uv; uv removes from the active project and environment when a " + "project is detected; pip mutates the environment." + ), ), yes: bool = typer.Option( False, @@ -175,7 +181,7 @@ def uninstall_command( help="Print the uninstall plan without mutating the current environment.", ), ) -> None: - """Uninstall one Data Designer plugin package, then verify package registration is removed.""" + """Uninstall one Data Designer plugin package, then verify declared runtime entry points are removed.""" controller = PluginCatalogController(DATA_DESIGNER_HOME) controller.run_uninstall( package, diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index df518c363..d972cc097 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -237,11 +237,11 @@ def run_install( raise typer.Exit(code=1) if self.install_service.verify_entry_points(package_entries): - print_success(f"Plugin package {entry.package.name!r} installed and registered") + print_success(f"Plugin package {entry.package.name!r} installed and runtime entry points verified") else: print_warning( f"Plugin package {entry.package.name!r} was installed, but Data Designer did not discover every " - "declared package entry point. Restart the shell or check the package entry point metadata." + "declared runtime entry point. Restart the shell or check the package entry point metadata." ) def run_uninstall( @@ -294,11 +294,11 @@ def run_uninstall( raise typer.Exit(code=1) if self.install_service.verify_entry_points_removed(package_entries): - print_success(f"Plugin package {entry.package.name!r} uninstalled and no longer registered") + print_success(f"Plugin package {entry.package.name!r} uninstalled and runtime entry points removed") else: print_warning( f"Plugin package {entry.package.name!r} was uninstalled, but Data Designer still discovers one or " - "more declared package entry points. Restart the shell or check the package environment." + "more declared runtime entry points. Restart the shell or check the package environment." ) def run_installed(self) -> None: diff --git a/packages/data-designer/src/data_designer/cli/main.py b/packages/data-designer/src/data_designer/cli/main.py index 0c46073f6..07b8a183b 100644 --- a/packages/data-designer/src/data_designer/cli/main.py +++ b/packages/data-designer/src/data_designer/cli/main.py @@ -165,12 +165,12 @@ def _is_version_request(args: list[str]) -> bool: "install": { "module": f"{_CMD}.plugin", "attr": "install_command", - "help": "Install a plugin package and verify package registration", + "help": "Install a plugin package and verify declared runtime entry points", }, "uninstall": { "module": f"{_CMD}.plugin", "attr": "uninstall_command", - "help": "Uninstall a plugin package and verify package registration is removed", + "help": "Uninstall a plugin package and verify declared runtime entry points are removed", }, "installed": { "module": f"{_CMD}.plugin", @@ -188,7 +188,7 @@ def plugin_callback( catalog: str | None = typer.Option( None, "--catalog", - help="Plugin catalog alias to use for catalog commands.", + help="Plugin catalog alias to use for commands that read package metadata.", ), ) -> None: _ = catalog diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index 07b5d273d..d798a2b14 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -176,7 +176,7 @@ def entries(self) -> list[PluginCatalogEntry]: @property def plugins(self) -> list[PluginCatalogEntry]: - """Backward-compatible alias for flattened runtime plugin entries.""" + """Convenience alias for flattened runtime plugin entries.""" return self.entries @@ -245,7 +245,7 @@ class UninstallPlan: @dataclass(frozen=True) class InstalledPluginInfo: - """Installed plugin entry point discovered without importing plugin code.""" + """Installed runtime plugin entry point discovered without importing plugin code.""" name: str entry_point_value: str diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index ca0e1b134..c16b969a4 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -163,7 +163,7 @@ def remove_catalog(self, alias: str) -> None: self.repository.remove_catalog(alias) def list_installed_plugins(self) -> list[InstalledPluginInfo]: - """List installed Data Designer plugin entry points without importing plugin modules.""" + """List installed Data Designer runtime plugin entry points without importing plugin modules.""" entry_points = importlib.metadata.entry_points(group=PLUGIN_ENTRY_POINT_GROUP) installed_plugins = [ InstalledPluginInfo(name=entry_point.name, entry_point_value=entry_point.value) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 209e16e6a..1274aa8bf 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -61,7 +61,7 @@ class _InstallTarget: class PluginInstallService: - """Resolve, execute, and verify plugin package install/uninstall plans. + """Resolve, execute, and verify plugin package install and uninstall plans. When no working directory is provided, plan resolution uses the current process directory at build time so CLI calls follow the user's active shell. @@ -160,11 +160,11 @@ def uninstall(self, plan: UninstallPlan) -> None: raise RuntimeError(f"Plugin package uninstaller exited with status {return_code}") def verify_entry_point(self, entry: PluginCatalogEntry) -> bool: - """Verify the plugin's declared entry point is installed.""" + """Verify the runtime plugin's declared entry point is installed.""" return self.verify_entry_points([entry]) def verify_entry_points(self, entries: list[PluginCatalogEntry]) -> bool: - """Verify every declared entry point for an installed catalog package.""" + """Verify every declared runtime entry point for an installed catalog package.""" if not entries: return False @@ -179,7 +179,7 @@ def verify_entry_points(self, entries: list[PluginCatalogEntry]) -> bool: ) def verify_entry_points_removed(self, entries: list[PluginCatalogEntry]) -> bool: - """Verify every declared entry point for a catalog package is no longer installed.""" + """Verify every declared runtime entry point for a catalog package is no longer installed.""" if not entries: return False diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index ba44a68c0..2e10e6846 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -468,7 +468,9 @@ def test_run_install_reports_success_when_verification_finds_entry_point( controller.install_service.install.assert_called_once_with(plan) controller.install_service.verify_entry_points.assert_called_once_with([entry]) - mock_print_success.assert_called_once_with("Plugin package 'data-designer-text-transform' installed and registered") + mock_print_success.assert_called_once_with( + "Plugin package 'data-designer-text-transform' installed and runtime entry points verified" + ) assert mock_console.print.call_count >= 1 @@ -494,7 +496,7 @@ def test_run_install_warns_when_verification_misses_entry_point( controller.install_service.verify_entry_points.assert_called_once_with([entry]) mock_print_warning.assert_called_once_with( "Plugin package 'data-designer-text-transform' was installed, but Data Designer did not discover every " - "declared package entry point. Restart the shell or check the package entry point metadata." + "declared runtime entry point. Restart the shell or check the package entry point metadata." ) assert mock_console.print.call_count >= 1 @@ -570,7 +572,7 @@ def test_run_uninstall_reports_success_when_entry_points_are_removed( controller.install_service.uninstall.assert_called_once_with(plan) controller.install_service.verify_entry_points_removed.assert_called_once_with([entry]) mock_print_success.assert_called_once_with( - "Plugin package 'data-designer-text-transform' uninstalled and no longer registered" + "Plugin package 'data-designer-text-transform' uninstalled and runtime entry points removed" ) assert mock_console.print.call_count >= 1 @@ -596,7 +598,7 @@ def test_run_uninstall_warns_when_entry_points_remain( controller.install_service.verify_entry_points_removed.assert_called_once_with([entry]) mock_print_warning.assert_called_once_with( "Plugin package 'data-designer-text-transform' was uninstalled, but Data Designer still discovers one or " - "more declared package entry points. Restart the shell or check the package environment." + "more declared runtime entry points. Restart the shell or check the package environment." ) assert mock_console.print.call_count >= 1 From 2b334cca22fcf3dc45b488843c2906507f6d7e9f Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 01:54:15 +0000 Subject: [PATCH 31/34] simplify plugin CLI docs --- architecture/cli.md | 31 +++++++------- .../src/data_designer/cli/README.md | 41 ++++++++++--------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/architecture/cli.md b/architecture/cli.md index 9bfba6377..26bc725c0 100644 --- a/architecture/cli.md +++ b/architecture/cli.md @@ -41,11 +41,11 @@ Plugin catalog commands use the same layering shape: | Layer | Role | Example | |-------|------|---------| | **Command** | Thin Typer entry, wires `DATA_DESIGNER_HOME` and command options | `plugin` subcommands (`list`, `search`, `info`, `install`, `uninstall`, `installed`, `catalog`) → `PluginCatalogController(DATA_DESIGNER_HOME)` | -| **Controller** | UX flow: catalog tables, package metadata, compatibility display, package mutation confirmations | `PluginCatalogController` composes catalog + install services | -| **Service** | Domain rules: package-first flattening, compatibility checks, install and uninstall planning, runtime entry-point verification | `PluginCatalogService`, `PluginInstallService` | +| **Controller** | UX flow: catalog tables, package metadata, compatibility display, install/uninstall confirmations | `PluginCatalogController` composes catalog + install services | +| **Service** | Domain rules: package listing, compatibility checks, uv/pip install and uninstall commands, plugin discovery verification | `PluginCatalogService`, `PluginInstallService` | | **Repository** | File/cache I/O for catalog aliases and catalog documents | `PluginCatalogRepository` | -The built-in `nvidia` catalog points at `https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`. `NVIDIA-NeMo/DataDesignerPlugins` defines the package-first catalog shape: top-level packages carry install metadata, compatibility constraints, docs, and nested runtime plugins. The CLI flattens nested plugins for list/search display, but `info`, `install`, and `uninstall` resolve package names or package aliases so environment mutations target the package distribution. Package aliases come from the `data-designer-{alias}` package-name pattern; for example, `data-designer-calculator` can be addressed as `calculator`. +The built-in `nvidia` catalog points at `https://nvidia-nemo.github.io/DataDesignerPlugins/catalog/plugins.json`. `NVIDIA-NeMo/DataDesignerPlugins` defines the catalog format. Each catalog entry is an installable package with docs, install metadata, compatibility constraints, and one or more runtime plugins. Users install and uninstall packages, not individual runtime plugins. Commands that take a package name also accept the package alias from the `data-designer-{alias}` package-name pattern; for example, `data-designer-calculator` can be addressed as `calculator`. ### Generation Commands @@ -79,7 +79,7 @@ User invokes command (e.g., `data-designer config models`) User invokes command (e.g., `data-designer plugin list`) → Command function wires DATA_DESIGNER_HOME and catalog options → PluginCatalogController resolves the catalog alias - → PluginCatalogService loads and filters package-first catalog entries + → PluginCatalogService loads packages and filters out incompatible packages by default → PluginCatalogRepository reads local config and cached/remote catalog JSON ``` @@ -88,22 +88,21 @@ User invokes command (e.g., `data-designer plugin list`) User invokes command (e.g., `data-designer plugin install calculator`) → PluginCatalogController resolves the plugin package name or package alias → PluginCatalogService evaluates Python and Data Designer compatibility - → PluginInstallService builds a pip/uv install plan for the package requirement. - In active uv projects it uses `uv add` so the package lands in `pyproject.toml`; - otherwise it mutates the active Python environment with `uv pip install` or pip. - The active Data Designer package family (`data-designer`, - `data-designer-config`, and `data-designer-engine`) is skipped, excluded, or - pinned from replacement depending on package-manager capabilities. - → PluginInstallService verifies declared runtime entry points after installation + → PluginInstallService chooses uv or pip and builds the command. + In active uv projects it uses `uv add` so the package is recorded in + `pyproject.toml`; otherwise it installs into the current Python environment. + Data Designer itself is already installed, so its packages are not reinstalled + or replaced while installing plugin dependencies. + → PluginInstallService verifies Data Designer can discover the package's runtime plugins ``` ``` User invokes command (e.g., `data-designer plugin uninstall calculator`) → PluginCatalogController resolves the plugin package name or package alias - → PluginInstallService builds a pip/uv uninstall plan for the package distribution. - Active uv projects remove the dependency from project metadata without a uv sync, - then uninstall the package from the active environment. - → PluginInstallService verifies declared runtime entry points are no longer discovered + → PluginInstallService chooses uv or pip and builds the uninstall command. + Active uv projects remove the dependency from project metadata and uninstall + the package from the current environment. + → PluginInstallService verifies Data Designer no longer discovers the package's runtime plugins ``` ### Generation @@ -119,7 +118,7 @@ User invokes command (e.g., `data-designer create config.yaml`) - **Lazy command loading** keeps `data-designer --help` responsive: command modules (and their heavy dependencies, such as the engine and model stacks) load only when a command is invoked, not at process startup. - **Controller/service/repo for setup workflows, direct API for generation** — config and plugin catalog workflows benefit from the layered pattern (testable services, swappable repositories). Generation doesn't need this indirection; it delegates to the same `DataDesigner` class that Python users call directly. - **`DATA_DESIGNER_HOME`** centralizes CLI-managed state (model configs, provider configs, MCP provider configs, tool configs, managed assets, plugin catalog aliases, and catalog caches) in a single directory, defaulting to `~/.data-designer/`. -- **Package-first plugin catalogs** keep install metadata at the package boundary while allowing one package to expose multiple runtime plugins through runtime entry points. +- **Package-first plugin catalogs** match how users install plugins: one package can provide one or more runtime plugins, but install and uninstall commands always target the package. - **Rich-based UI** provides formatted tables, progress bars, and interactive prompts without requiring a web interface. ## Cross-References diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index d865810e4..52e42048c 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -10,8 +10,8 @@ The CLI provides an interactive interface for managing: - **MCP Providers**: MCP server configurations for tool integration - **Tool Configs**: Tool definitions used by configured models and workflows - **Managed Assets**: Persona dataset downloads under the Data Designer home directory -- **Plugin Catalogs**: Catalog aliases for discovering Data Designer plugin packages -- **Plugin Installs**: Safe install-plan rendering, package-manager execution with active Data Designer package-family protection, and runtime entry-point verification +- **Plugin Catalogs**: Catalog aliases for finding Data Designer plugin packages +- **Plugin Packages**: Install and uninstall packages from catalogs, check version compatibility first, and verify Data Designer can discover the plugins they provide Configuration files and CLI-managed state are stored in `~/.data-designer/` by default. @@ -93,8 +93,8 @@ The CLI follows a **layered architecture** pattern, separating concerns into dis - `mcp_provider_service.py`: MCP provider configuration business logic - `model_service.py`: Model configuration business logic - `provider_service.py`: Provider business logic - - `plugin_catalog_service.py`: Plugin catalog discovery, search, compatibility checks, and installed runtime entry-point listing - - `plugin_install_service.py`: Plugin install and uninstall plan resolution, package-manager execution, active Data Designer package-family protection, and runtime entry-point verification + - `plugin_catalog_service.py`: Plugin catalog loading, search, compatibility checks, and installed plugin listing + - `plugin_install_service.py`: Chooses and runs uv or pip commands for installing/uninstalling plugin packages, keeps installed Data Designer packages in place, and verifies installed plugins - `tool_service.py`: Tool configuration business logic **Key Methods**: @@ -338,19 +338,19 @@ data-designer plugin list # Search a specific catalog data-designer plugin --catalog research search transform -# Show metadata, compatibility, docs, and the install plan +# Show package metadata, compatibility, docs, and the install command data-designer plugin info github -# Install a plugin package from a catalog and verify declared runtime entry points +# Install a plugin package from a catalog and verify Data Designer can discover its plugins data-designer plugin install github --yes -# Preview the install plan without mutating the environment +# Preview without changing the current environment data-designer plugin install github --dry-run -# Uninstall a plugin package from a catalog and verify declared runtime entry points are removed +# Uninstall a plugin package and verify Data Designer no longer discovers its plugins data-designer plugin uninstall github --yes -# Preview the uninstall plan without mutating the environment +# Preview without changing the current environment data-designer plugin uninstall github --dry-run # Add and manage catalog aliases @@ -362,14 +362,15 @@ data-designer plugin catalog remove research data-designer plugin installed ``` -Install plans protect the active Data Designer package family (`data-designer`, -`data-designer-config`, and `data-designer-engine`) before invoking the package -manager. The plugin package and its other dependencies are resolved normally, -but the installed Data Designer packages are kept from being replaced by plugin -package dependencies. In an active virtual environment with a user -`pyproject.toml`, `uv` uses `uv add` so the plugin package is recorded in the -project; otherwise it uses `uv pip install`. `uv` plugin installs require -`uv >= 0.6.0`; auto mode falls back to `pip` when `uv` is missing or too old. -`pip` remains supported for pip-only environments. `uv` project installs skip -installing the Data Designer package family; pip installs use a process-scoped -temporary constraint file because pip constraints are file-based. +When installing a plugin package, the CLI first checks the package's Python and +Data Designer version requirements. The plugin package and its other +dependencies are installed normally, but the currently installed Data Designer +packages (`data-designer`, `data-designer-config`, and `data-designer-engine`) +are kept in place. This prevents a plugin dependency from upgrading, +downgrading, or reinstalling Data Designer itself. + +In an active virtual environment with a user `pyproject.toml`, `uv` uses +`uv add` so the plugin package is recorded in the project. Otherwise the CLI +installs into the current Python environment with `uv pip install` or `pip`. +`uv` plugin installs require `uv >= 0.6.0`; auto mode falls back to `pip` when +`uv` is missing or too old. `pip` remains supported for pip-only environments. From 66c74a5d26f9097ffa617af9ba83df66baeff36a Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 02:02:14 +0000 Subject: [PATCH 32/34] narrow plugin search fields --- .../src/data_designer/cli/commands/plugin.py | 2 +- .../cli/services/plugin_catalog_service.py | 13 ++++------- .../services/test_plugin_catalog_service.py | 22 ++++++++++++++++++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/commands/plugin.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py index eaf5940b7..6da72a692 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugin.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -41,7 +41,7 @@ def list_command( def search_command( ctx: typer.Context, query: str = typer.Argument( - help="Keyword, runtime plugin name or type, package name, requirement, docs URL, or entry point to search for." + help="Keyword, package name or alias, description, runtime plugin name, or runtime plugin type to search for." ), catalog: str | None = typer.Option( None, diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index c16b969a4..2c9da8548 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -61,7 +61,7 @@ def search_entries( refresh: bool = False, include_incompatible: bool = False, ) -> list[PluginCatalogEntry]: - """Search catalog entries by simple token matching.""" + """Search catalog entries by package metadata and runtime plugin metadata.""" query_tokens = _tokenize(query) if not query_tokens: return [] @@ -217,16 +217,11 @@ def _tokenize(value: str) -> list[str]: def _entry_search_text(entry: PluginCatalogEntry) -> str: package_name = canonicalize_name(entry.package.name) values = [ - entry.name, - entry.plugin_type.value, - entry.description, entry.package.name, _package_alias(package_name) or "", - entry.install.requirement, - entry.install.index_url or "", - entry.entry_point.name, - entry.entry_point.value, - entry.docs.url if entry.docs is not None and entry.docs.url else "", + entry.description, + entry.name, + entry.plugin_type.value, ] return " ".join(values).lower() diff --git a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py index 75952bf24..d318f571f 100644 --- a/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py +++ b/packages/data-designer/tests/cli/services/test_plugin_catalog_service.py @@ -35,7 +35,7 @@ def test_list_entries_filters_incompatible_plugins_by_default(tmp_path: Path) -> ] -def test_search_entries_matches_name_type_package_and_docs(tmp_path: Path) -> None: +def test_search_entries_matches_package_description_name_and_type(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") @@ -48,6 +48,26 @@ def test_search_entries_matches_name_type_package_and_docs(tmp_path: Path) -> No assert [entry.name for entry in type_matches] == ["compatible-plugin"] +def test_search_entries_ignores_install_docs_and_entry_point_metadata(tmp_path: Path) -> None: + package = _package( + package_name="data-designer-retrieval-sdg", + data_designer_specifier=">=0.5.7", + plugins=[_runtime_plugin(name="document-chunker", plugin_type="seed-reader")], + ) + package["install"]["index_url"] = "https://nvidia-nemo.github.io/DataDesignerPlugins/simple/" + package["docs"]["url"] = "https://nvidia-nemo.github.io/DataDesignerPlugins/plugins/data-designer-retrieval-sdg/" + package["plugins"][0]["entry_point"]["value"] = "data_designer_github_noise.plugin:plugin" + catalog_path = tmp_path / "plugins.json" + catalog_path.write_text(json.dumps({"schema_version": 2, "packages": [package]})) + repository = PluginCatalogRepository(tmp_path) + repository.add_catalog("local", str(catalog_path)) + service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") + + matches = service.search_entries("github", "local") + + assert matches == [] + + def test_evaluate_compatibility_reports_data_designer_constraint(tmp_path: Path) -> None: repository = _repository_with_catalog(tmp_path) service = PluginCatalogService(repository, python_version="3.11.0", data_designer_version="0.5.7") From 0574e7bda186b402a848dfd02e78064e8821e966 Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 02:14:15 +0000 Subject: [PATCH 33/34] hide plugin catalog cache ttl --- .../data-designer/src/data_designer/cli/README.md | 1 - .../src/data_designer/cli/commands/plugin.py | 7 ------- .../cli/controllers/plugin_catalog_controller.py | 4 ---- .../cli/repositories/plugin_catalog_repository.py | 2 +- .../cli/services/plugin_catalog_service.py | 2 -- .../tests/cli/commands/test_plugin_command.py | 3 --- .../cli/controllers/test_plugin_catalog_controller.py | 1 - .../repositories/test_plugin_catalog_repository.py | 11 +++++++++++ 8 files changed, 12 insertions(+), 19 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 52e42048c..2494ab004 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -274,7 +274,6 @@ catalogs: - alias: research url: https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json trusted: false - cache_ttl_seconds: 86400 ``` ### `~/.data-designer/plugin-catalog-cache/` diff --git a/packages/data-designer/src/data_designer/cli/commands/plugin.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py index 6da72a692..7cc6eeacd 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugin.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -218,12 +218,6 @@ def catalog_add_command( "--trusted", help="Mark the catalog as trusted for install-plan display and confirmations.", ), - cache_ttl_seconds: int = typer.Option( - 24 * 60 * 60, - "--cache-ttl-seconds", - min=0, - help="Seconds before cached catalog metadata is refreshed. Use 0 to always refresh.", - ), ) -> None: """Add a plugin catalog alias.""" _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") @@ -232,7 +226,6 @@ def catalog_add_command( alias=alias, url=url, trusted=trusted, - cache_ttl_seconds=cache_ttl_seconds, ) diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index d972cc097..405402f75 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -323,14 +323,12 @@ def run_catalog_list(self) -> None: table.add_column("Alias", style=NordColor.NORD14.value, no_wrap=True) table.add_column("URL", style=NordColor.NORD4.value) table.add_column("Trusted", style=NordColor.NORD13.value, justify="center") - table.add_column("Cache TTL", style=NordColor.NORD9.value, justify="right") for catalog in catalogs: table.add_row( catalog.alias, catalog.url, "yes" if catalog.trusted else "no", - f"{catalog.cache_ttl_seconds}s", ) console.print(table) @@ -340,7 +338,6 @@ def run_catalog_add( alias: str, url: str, trusted: bool, - cache_ttl_seconds: int, ) -> None: """Add a plugin catalog alias.""" try: @@ -348,7 +345,6 @@ def run_catalog_add( alias, url, trusted=trusted, - cache_ttl_seconds=cache_ttl_seconds, ) except ValidationError as e: if any(tuple(error["loc"]) == ("alias",) for error in e.errors()): diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py index 4d3a230e7..d2c38f134 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py @@ -59,7 +59,7 @@ def load(self) -> PluginCatalogRegistry | None: def save(self, config: PluginCatalogRegistry) -> None: """Save user-configured plugin catalogs.""" - config_dict = config.model_dump(mode="json", exclude_none=True) + config_dict = config.model_dump(mode="json", exclude_none=True, exclude_defaults=True) save_config_file(self.config_file, config_dict) def list_catalogs(self) -> list[PluginCatalogConfig]: diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index 2c9da8548..8f376ad4c 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -148,14 +148,12 @@ def add_catalog( url: str, *, trusted: bool, - cache_ttl_seconds: int, ) -> PluginCatalogConfig: """Add a plugin catalog alias.""" return self.repository.add_catalog( alias, url, trusted=trusted, - cache_ttl_seconds=cache_ttl_seconds, ) def remove_catalog(self, alias: str) -> None: diff --git a/packages/data-designer/tests/cli/commands/test_plugin_command.py b/packages/data-designer/tests/cli/commands/test_plugin_command.py index 1ab244bc7..6a7eb60b3 100644 --- a/packages/data-designer/tests/cli/commands/test_plugin_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugin_command.py @@ -128,8 +128,6 @@ def test_plugin_catalog_add_command_delegates_to_controller(mock_ctrl_cls: Magic "research", "https://github.com/acme/dd-plugins", "--trusted", - "--cache-ttl-seconds", - "60", ], ) @@ -138,7 +136,6 @@ def test_plugin_catalog_add_command_delegates_to_controller(mock_ctrl_cls: Magic alias="research", url="https://github.com/acme/dd-plugins", trusted=True, - cache_ttl_seconds=60, ) diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index 2e10e6846..e3e1ff812 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -615,7 +615,6 @@ def test_run_catalog_add_wraps_invalid_alias_validation_error( alias="foo/bar", url="https://github.com/acme/dd-plugins", trusted=False, - cache_ttl_seconds=60, ) assert exc_info.value.exit_code == 1 diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py index e437daf0b..dcc8647d1 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py @@ -52,6 +52,17 @@ def test_add_catalog_normalizes_github_repository_url(tmp_path: Path) -> None: assert repository.get_catalog("research") == catalog +def test_add_catalog_persists_only_public_catalog_fields(tmp_path: Path) -> None: + repository = PluginCatalogRepository(tmp_path) + + repository.add_catalog("research", "https://github.com/acme/dd-plugins") + + saved_registry = repository.config_file.read_text() + assert "alias: research" in saved_registry + assert "url: https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json" in saved_registry + assert "cache_ttl_seconds" not in saved_registry + + def test_add_catalog_normalizes_github_tree_url_with_subdirectory(tmp_path: Path) -> None: repository = PluginCatalogRepository(tmp_path) From 2a91d2105b79a9b33cdbf9e31718dbbfb5cd6b8a Mon Sep 17 00:00:00 2001 From: Johnny Greco Date: Sat, 9 May 2026 02:18:37 +0000 Subject: [PATCH 34/34] remove plugin catalog trust flag --- .../src/data_designer/cli/README.md | 1 - .../src/data_designer/cli/commands/plugin.py | 6 -- .../controllers/plugin_catalog_controller.py | 10 --- .../src/data_designer/cli/plugin_catalog.py | 4 +- .../repositories/plugin_catalog_repository.py | 4 -- .../cli/services/plugin_catalog_service.py | 3 - .../cli/services/plugin_install_service.py | 1 - .../tests/cli/commands/test_plugin_command.py | 2 - .../test_plugin_catalog_controller.py | 64 ++++++------------- .../test_plugin_catalog_repository.py | 2 - 10 files changed, 20 insertions(+), 77 deletions(-) diff --git a/packages/data-designer/src/data_designer/cli/README.md b/packages/data-designer/src/data_designer/cli/README.md index 2494ab004..b5f6f087a 100644 --- a/packages/data-designer/src/data_designer/cli/README.md +++ b/packages/data-designer/src/data_designer/cli/README.md @@ -273,7 +273,6 @@ staging. catalogs: - alias: research url: https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json - trusted: false ``` ### `~/.data-designer/plugin-catalog-cache/` diff --git a/packages/data-designer/src/data_designer/cli/commands/plugin.py b/packages/data-designer/src/data_designer/cli/commands/plugin.py index 7cc6eeacd..a9dbf974e 100644 --- a/packages/data-designer/src/data_designer/cli/commands/plugin.py +++ b/packages/data-designer/src/data_designer/cli/commands/plugin.py @@ -213,11 +213,6 @@ def catalog_add_command( url: str = typer.Argument( help="Catalog repository URL, catalog URL, local catalog file, or local catalog directory." ), - trusted: bool = typer.Option( - False, - "--trusted", - help="Mark the catalog as trusted for install-plan display and confirmations.", - ), ) -> None: """Add a plugin catalog alias.""" _warn_if_parent_catalog_unused(ctx, "catalog management commands operate on aliases directly") @@ -225,7 +220,6 @@ def catalog_add_command( controller.run_catalog_add( alias=alias, url=url, - trusted=trusted, ) diff --git a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py index 405402f75..4b5777813 100644 --- a/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py +++ b/packages/data-designer/src/data_designer/cli/controllers/plugin_catalog_controller.py @@ -207,12 +207,6 @@ def run_install( if plan.source_warning is not None: print_warning(plan.source_warning) - if not catalog.trusted: - print_warning( - "This catalog is not marked trusted. Plugin package installation executes Python package code from " - "the requirement above." - ) - if dry_run: if not compatibility.is_compatible: print_warning( @@ -322,13 +316,11 @@ def run_catalog_list(self) -> None: table = Table(title="Plugin Catalogs", border_style=NordColor.NORD8.value) table.add_column("Alias", style=NordColor.NORD14.value, no_wrap=True) table.add_column("URL", style=NordColor.NORD4.value) - table.add_column("Trusted", style=NordColor.NORD13.value, justify="center") for catalog in catalogs: table.add_row( catalog.alias, catalog.url, - "yes" if catalog.trusted else "no", ) console.print(table) @@ -337,14 +329,12 @@ def run_catalog_add( *, alias: str, url: str, - trusted: bool, ) -> None: """Add a plugin catalog alias.""" try: catalog = self.catalog_service.add_catalog( alias, url, - trusted=trusted, ) except ValidationError as e: if any(tuple(error["loc"]) == ("alias",) for error in e.errors()): diff --git a/packages/data-designer/src/data_designer/cli/plugin_catalog.py b/packages/data-designer/src/data_designer/cli/plugin_catalog.py index d798a2b14..4cfe32e03 100644 --- a/packages/data-designer/src/data_designer/cli/plugin_catalog.py +++ b/packages/data-designer/src/data_designer/cli/plugin_catalog.py @@ -185,7 +185,6 @@ class PluginCatalogConfig(BaseModel): alias: str = Field(pattern=PLUGIN_CATALOG_ALIAS_PATTERN) url: str - trusted: bool = False cache_ttl_seconds: int = Field(default=PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, ge=0) @@ -221,7 +220,6 @@ class InstallPlan: command: list[str] manager: str catalog_alias: str - trusted_catalog: bool source_warning: str | None = None data_designer_protection: str | None = None command_stdin: str | None = None @@ -428,7 +426,7 @@ def _validate_catalog_object_keys(context: str, value: dict[str, object], expect keys = set(value) if keys != expected_keys: # Catalog v2 is strict by design: additive wire-schema changes should bump - # schema_version so older CLIs do not silently ignore trust-sensitive fields. + # schema_version so older CLIs do not silently ignore new fields. raise PluginCatalogError( f"{context} has invalid fields; expected {{{_format_catalog_keys(expected_keys)}}}, " f"got {{{_format_catalog_keys(keys)}}}" diff --git a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py index d2c38f134..bc5b14ae3 100644 --- a/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py +++ b/packages/data-designer/src/data_designer/cli/repositories/plugin_catalog_repository.py @@ -16,7 +16,6 @@ from data_designer.cli.plugin_catalog import ( DEFAULT_PLUGIN_CATALOG_ALIAS, - DEFAULT_PLUGIN_CATALOG_URL, MAX_PLUGIN_CATALOG_SIZE_BYTES, PLUGIN_CATALOG_CACHE_DIR_NAME, PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, @@ -80,7 +79,6 @@ def add_catalog( alias: str, url: str, *, - trusted: bool = False, cache_ttl_seconds: int = PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, ) -> PluginCatalogConfig: """Persist a new catalog alias. @@ -94,7 +92,6 @@ def add_catalog( catalog = PluginCatalogConfig( alias=alias, url=normalize_catalog_location(url), - trusted=trusted, cache_ttl_seconds=cache_ttl_seconds, ) registry = self.load() or PluginCatalogRegistry() @@ -235,7 +232,6 @@ def default_catalog() -> PluginCatalogConfig: return PluginCatalogConfig( alias=DEFAULT_PLUGIN_CATALOG_ALIAS, url=catalog_url, - trusted=catalog_url == DEFAULT_PLUGIN_CATALOG_URL, cache_ttl_seconds=PLUGIN_CATALOG_DEFAULT_CACHE_TTL_SECONDS, ) diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py index 8f376ad4c..f4f992896 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_catalog_service.py @@ -146,14 +146,11 @@ def add_catalog( self, alias: str, url: str, - *, - trusted: bool, ) -> PluginCatalogConfig: """Add a plugin catalog alias.""" return self.repository.add_catalog( alias, url, - trusted=trusted, ) def remove_catalog(self, alias: str) -> None: diff --git a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py index 1274aa8bf..09545e6bb 100644 --- a/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py +++ b/packages/data-designer/src/data_designer/cli/services/plugin_install_service.py @@ -104,7 +104,6 @@ def build_install_plan( command=command, manager=target.manager, catalog_alias=catalog.alias, - trusted_catalog=catalog.trusted, source_warning=_combine_warnings(target.warning, source_warning), data_designer_protection=data_designer_protection, command_stdin=command_stdin, diff --git a/packages/data-designer/tests/cli/commands/test_plugin_command.py b/packages/data-designer/tests/cli/commands/test_plugin_command.py index 6a7eb60b3..19ac5102b 100644 --- a/packages/data-designer/tests/cli/commands/test_plugin_command.py +++ b/packages/data-designer/tests/cli/commands/test_plugin_command.py @@ -127,7 +127,6 @@ def test_plugin_catalog_add_command_delegates_to_controller(mock_ctrl_cls: Magic "add", "research", "https://github.com/acme/dd-plugins", - "--trusted", ], ) @@ -135,7 +134,6 @@ def test_plugin_catalog_add_command_delegates_to_controller(mock_ctrl_cls: Magic mock_ctrl.run_catalog_add.assert_called_once_with( alias="research", url="https://github.com/acme/dd-plugins", - trusted=True, ) diff --git a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py index e3e1ff812..a21e0185f 100644 --- a/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py +++ b/packages/data-designer/tests/cli/controllers/test_plugin_catalog_controller.py @@ -42,7 +42,7 @@ def test_run_list_mentions_hidden_incompatible_packages_when_visible_list_is_emp controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.list_entries.side_effect = [[], [entry]] @@ -69,7 +69,7 @@ def test_run_search_mentions_hidden_incompatible_packages_when_visible_matches_a controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.search_entries.side_effect = [[], [entry]] @@ -95,7 +95,7 @@ def test_run_list_renders_package_first_catalog_table( _entry(name="text-column", plugin_type="column-generator"), _entry(name="text-processor", plugin_type="processor"), ] - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.list_entries.return_value = package_entries controller.catalog_service.group_entries_by_package.return_value = { @@ -148,7 +148,7 @@ def test_run_info_renders_package_metadata_with_nested_runtime_plugins( _entry(name="text-column", plugin_type="column-generator"), _entry(name="text-processor", plugin_type="processor"), ] - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = package_entries controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) @@ -209,7 +209,7 @@ def test_run_info_warns_when_install_plan_has_source_warning( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) @@ -230,7 +230,7 @@ def test_run_info_rejects_runtime_plugin_name_that_is_not_package_alias( mock_print_error: MagicMock, controller: PluginCatalogController, ) -> None: - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [] @@ -255,7 +255,7 @@ def test_run_install_dry_run_renders_plan_without_installing( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() plan = _plan(catalog, data_designer_protection="pinned installed Data Designer packages; data-designer 0.5.10") controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] @@ -288,7 +288,7 @@ def test_run_install_blocks_incompatible_package( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( @@ -318,7 +318,7 @@ def test_run_install_rejects_runtime_plugin_name_as_target( mock_print_error: MagicMock, controller: PluginCatalogController, ) -> None: - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [] @@ -344,7 +344,7 @@ def test_run_install_dry_run_renders_incompatible_plan_and_block_message( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( @@ -376,7 +376,7 @@ def test_run_install_dry_run_allows_incompatible_entry_for_inspection( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult( @@ -410,7 +410,7 @@ def test_run_install_warns_when_install_plan_has_source_warning( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) @@ -425,29 +425,6 @@ def test_run_install_warns_when_install_plan_has_source_warning( assert mock_console.print.call_count >= 1 -@patch("data_designer.cli.controllers.plugin_catalog_controller.console") -@patch("data_designer.cli.controllers.plugin_catalog_controller.print_warning") -def test_run_install_warns_for_untrusted_catalog( - mock_print_warning: MagicMock, - mock_console: MagicMock, - controller: PluginCatalogController, -) -> None: - entry = _entry() - catalog = _catalog(trusted=False) - controller.catalog_service.get_catalog.return_value = catalog - controller.catalog_service.get_package_entries.return_value = [entry] - controller.catalog_service.evaluate_compatibility.return_value = CompatibilityResult(True, []) - controller.install_service.build_install_plan.return_value = _plan(catalog) - - controller.run_install("data-designer-text-transform", catalog_alias="local", dry_run=True) - - mock_print_warning.assert_called_once_with( - "This catalog is not marked trusted. Plugin package installation executes Python package code from " - "the requirement above." - ) - assert mock_console.print.call_count >= 1 - - @patch("data_designer.cli.controllers.plugin_catalog_controller.console") @patch("data_designer.cli.controllers.plugin_catalog_controller.print_success") def test_run_install_reports_success_when_verification_finds_entry_point( @@ -456,7 +433,7 @@ def test_run_install_reports_success_when_verification_finds_entry_point( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() plan = _plan(catalog) controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] @@ -482,7 +459,7 @@ def test_run_install_warns_when_verification_misses_entry_point( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() plan = _plan(catalog) controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] @@ -509,7 +486,7 @@ def test_run_uninstall_dry_run_renders_plan_without_uninstalling( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() plan = _uninstall_plan(catalog) controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] @@ -539,7 +516,7 @@ def test_run_uninstall_wraps_plan_error( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] controller.install_service.build_uninstall_plan.side_effect = ValueError("uv was requested") @@ -560,7 +537,7 @@ def test_run_uninstall_reports_success_when_entry_points_are_removed( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() plan = _uninstall_plan(catalog) controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] @@ -585,7 +562,7 @@ def test_run_uninstall_warns_when_entry_points_remain( controller: PluginCatalogController, ) -> None: entry = _entry() - catalog = _catalog(trusted=True) + catalog = _catalog() plan = _uninstall_plan(catalog) controller.catalog_service.get_catalog.return_value = catalog controller.catalog_service.get_package_entries.return_value = [entry] @@ -614,7 +591,6 @@ def test_run_catalog_add_wraps_invalid_alias_validation_error( plugin_controller.run_catalog_add( alias="foo/bar", url="https://github.com/acme/dd-plugins", - trusted=False, ) assert exc_info.value.exit_code == 1 @@ -635,11 +611,10 @@ def test_run_catalog_list_wraps_registry_load_error( mock_print_error.assert_called_once_with("Failed to list plugin catalogs: bad registry") -def _catalog(*, trusted: bool) -> PluginCatalogConfig: +def _catalog() -> PluginCatalogConfig: return PluginCatalogConfig( alias="local", url="https://raw.githubusercontent.com/acme/dd-plugins/main/catalog/plugins.json", - trusted=trusted, ) @@ -655,7 +630,6 @@ def _plan( command=["python", "-m", "pip", "install", "data-designer-text-transform"], manager="pip", catalog_alias=catalog.alias, - trusted_catalog=catalog.trusted, source_warning=source_warning, data_designer_protection=data_designer_protection, ) diff --git a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py index dcc8647d1..86dc7898f 100644 --- a/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py +++ b/packages/data-designer/tests/cli/repositories/test_plugin_catalog_repository.py @@ -30,7 +30,6 @@ def test_repository_includes_default_nvidia_catalog(tmp_path: Path) -> None: catalogs = repository.list_catalogs() assert [catalog.alias for catalog in catalogs] == [DEFAULT_PLUGIN_CATALOG_ALIAS] - assert catalogs[0].trusted is True def test_default_catalog_honors_url_environment_override(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -40,7 +39,6 @@ def test_default_catalog_honors_url_environment_override(tmp_path: Path, monkeyp catalog = repository.default_catalog() assert catalog.url == "https://example.test/catalog/plugins.json" - assert catalog.trusted is False def test_add_catalog_normalizes_github_repository_url(tmp_path: Path) -> None: