From 2ffcbd7bf1469e50ef061808d9bf9fce5033755b Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 12 Feb 2026 20:35:33 -0500 Subject: [PATCH 01/24] feat(otdf-sdk-mgr): add SDK version management CLI and refactor scripts Add new SDK management package for version resolution, artifact installation, and checkout. Refactor shell scripts to thin wrappers delegating to otdf-sdk-mgr. Update CI workflows with step IDs, conditional guards, and otdf-sdk-mgr version resolution. Co-Authored-By: Claude Opus 4.6 Signed-off-by: David Mihalcik --- .github/workflows/check.yml | 4 +- .github/workflows/xtest.yml | 26 +- AGENTS.md | 14 +- otdf-sdk-mgr/README.md | 84 ++++ otdf-sdk-mgr/pyproject.toml | 31 ++ otdf-sdk-mgr/src/otdf_sdk_mgr/__init__.py | 3 + otdf-sdk-mgr/src/otdf_sdk_mgr/__main__.py | 5 + otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 74 ++++ otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py | 89 ++++ otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py | 80 ++++ otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 129 ++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 100 +++++ otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 216 ++++++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py | 92 +++++ otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 178 ++++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 301 ++++++++++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py | 38 ++ otdf-sdk-mgr/uv.lock | 263 ++++++++++++ xtest/sdk/go/cli.sh | 7 +- xtest/sdk/go/otdfctl.sh | 7 +- xtest/sdk/js/cli.sh | 2 +- xtest/sdk/scripts/checkout-all.sh | 19 +- xtest/sdk/scripts/checkout-sdk-branch.sh | 75 +--- xtest/sdk/scripts/cleanup-all.sh | 31 +- xtest/sdk/scripts/list-versions.py | 139 +++++++ xtest/sdk/scripts/post-checkout-java.sh | 100 +---- xtest/sdk/scripts/requirements.txt | 2 + xtest/sdk/scripts/resolve-version.py | 387 +----------------- xtest/setup-cli-tool/action.yaml | 42 +- 29 files changed, 1976 insertions(+), 562 deletions(-) create mode 100644 otdf-sdk-mgr/README.md create mode 100644 otdf-sdk-mgr/pyproject.toml create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/__init__.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/__main__.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/config.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py create mode 100644 otdf-sdk-mgr/uv.lock create mode 100755 xtest/sdk/scripts/list-versions.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fe6215b4..c147d24d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,11 +34,11 @@ jobs: uv run ruff format --check . uv run pyright working-directory: xtest - - name: Lint and test otdf-local + - name: Lint and test otdf-sdk-mgr run: | uv sync uv run ruff check . uv run ruff format --check . uv run pyright uv run pytest --maxfail=1 --disable-warnings -v --tb=short -m "not integration" - working-directory: otdf-local + working-directory: otdf-sdk-mgr diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index c691aeec..37086693 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -117,9 +117,7 @@ jobs: - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b with: python-version: "3.14" - - run: |- - pip install -r scripts/requirements.txt - working-directory: otdf-sdk/xtest/sdk + - uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0 - id: version-info uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1 with: @@ -134,8 +132,7 @@ jobs: const { execSync } = require('child_process'); const path = require('path'); - const workingDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk/xtest/sdk'); - const resolveVersionScript = path.join(workingDir, 'scripts/resolve-version.py'); + const sdkMgrDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk/otdf-sdk-mgr'); const defaultTags = process.env.DEFAULT_TAGS || 'main'; core.setOutput('default-tags', defaultTags); @@ -150,7 +147,7 @@ jobs: for (const [sdkType, ref] of Object.entries(refs)) { try { - const output = execSync(`python3 ${resolveVersionScript} ${sdkType} ${ref}`, { cwd: workingDir }).toString(); + const output = execSync(`uv run --project ${sdkMgrDir} otdf-sdk-mgr versions resolve ${sdkType} ${ref}`, { cwd: sdkMgrDir }).toString(); const ojson = JSON.parse(output); if (!!ojson.err) { throw new Error(ojson.err); @@ -269,6 +266,7 @@ jobs: ######### CHECKOUT JS CLI ############# - name: Configure js-sdk + id: configure-js uses: ./otdftests/xtest/setup-cli-tool with: path: otdftests/xtest/sdk @@ -276,6 +274,7 @@ jobs: version-info: "${{ needs.resolve-versions.outputs.js }}" - name: Cache npm + if: fromJson(steps.configure-js.outputs.heads)[0] != null uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 with: path: ~/.npm @@ -286,6 +285,7 @@ jobs: ######## SETUP THE JS CLI ############# - name: build and setup the web-sdk cli id: build-web-sdk + if: fromJson(steps.configure-js.outputs.heads)[0] != null run: | make working-directory: otdftests/xtest/sdk/js @@ -300,6 +300,7 @@ jobs: version-info: "${{ needs.resolve-versions.outputs.go }}" - name: Cache Go modules + if: fromJson(steps.configure-go.outputs.heads)[0] != null uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 with: path: | @@ -311,13 +312,14 @@ jobs: - name: Resolve otdfctl heads id: resolve-otdfctl-heads + if: fromJson(steps.configure-go.outputs.heads)[0] != null run: |- echo "OTDFCTL_HEADS=$OTDFCTL_HEADS" >> "$GITHUB_ENV" env: OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} - name: Replace otdfctl go.mod packages, but only at head version of platform - if: env.FOCUS_SDK == 'go' && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) + if: fromJson(steps.configure-go.outputs.heads)[0] != null && env.FOCUS_SDK == 'go' && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) env: PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} run: |- @@ -337,6 +339,7 @@ jobs: ######## SETUP THE GO CLI ############# - name: Prepare go cli + if: fromJson(steps.configure-go.outputs.heads)[0] != null run: |- make working-directory: otdftests/xtest/sdk/go @@ -344,6 +347,7 @@ jobs: ####### CHECKOUT JAVA SDK ############## - name: Configure java-sdk + id: configure-java uses: ./otdftests/xtest/setup-cli-tool with: path: otdftests/xtest/sdk @@ -351,6 +355,7 @@ jobs: version-info: "${{ needs.resolve-versions.outputs.java }}" - name: Cache Maven repository + if: fromJson(steps.configure-java.outputs.heads)[0] != null uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 with: path: ~/.m2/repository @@ -359,8 +364,10 @@ jobs: maven-${{ runner.os }}- - name: pre-release protocol buffers for java-sdk - if: | - (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) + if: >- + fromJson(steps.configure-java.outputs.heads)[0] != null + && (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') + && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) run: |- echo "Replacing .env files for java-sdk..." echo "Platform tag: $platform_tag" @@ -382,6 +389,7 @@ jobs: ####### SETUP JAVA CLI ############## - name: Prepare java cli + if: fromJson(steps.configure-java.outputs.heads)[0] != null run: | make working-directory: otdftests/xtest/sdk/java diff --git a/AGENTS.md b/AGENTS.md index 5990efe7..9165087a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,10 +6,22 @@ This guide provides essential knowledge for AI agents performing updates, refact ### Structure - **Test Directory**: `xtest/` - pytest-based integration tests -- **SDK Distributions**: `xtest/sdk/{go,java,js}/` - SDK checkout, build, and CLI wrappers +- **SDK Distributions**: `sdk/{go,java,js}/dist/` - built SDK distributions with CLI wrappers +- **SDK Configuration**: `otdf-sdk-mgr install` - installs SDK CLIs from released artifacts or delegates to source builds +- **SDK Version Lookup**: `otdf-sdk-mgr versions list` - lists released artifacts across registries (Go git tags, npm, Maven Central, GitHub Releases) - **Platform**: `platform/` - OpenTDF platform service - **Test Runner**: pytest with custom CLI options +### Configuring SDK Artifacts + +Use `otdf-sdk-mgr` (uv-managed CLI in `tests/otdf-sdk-mgr/`) to install SDK CLIs from released artifacts or source. See `otdf-sdk-mgr/README.md` for full command reference. + +```bash +cd tests/otdf-sdk-mgr && uv tool install --editable . +otdf-sdk-mgr install stable # Latest stable releases (recommended) +otdf-sdk-mgr install tip go # Build from source +``` + ### Running Tests ```bash diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md new file mode 100644 index 00000000..bdc7abee --- /dev/null +++ b/otdf-sdk-mgr/README.md @@ -0,0 +1,84 @@ +# otdf-sdk-mgr + +SDK artifact management CLI for OpenTDF cross-client tests. Installs SDK CLIs from **released artifacts** (fast, deterministic) or **source** (for branch/PR testing). Both modes produce the same `sdk/{go,java,js}/dist/{version}/` directory structure. + +## Installation + +```bash +cd tests/otdf-sdk-mgr && uv tool install --editable . +``` + +## Commands + +### install + +```bash +# Install latest stable releases for all SDKs (recommended for local testing) +otdf-sdk-mgr install stable + +# Install LTS versions +otdf-sdk-mgr install lts + +# Install specific released versions +otdf-sdk-mgr install release go:v0.24.0 js:0.4.0 java:v0.9.0 + +# Install from tip of main branch (source build) +otdf-sdk-mgr install tip +otdf-sdk-mgr install tip go # Single SDK + +# Install a published version with optional dist name (defaults to version tag) +otdf-sdk-mgr install artifact --sdk go --version v0.24.0 +otdf-sdk-mgr install artifact --sdk go --version v0.24.0 --dist-name my-tag +``` + +### versions + +```bash +# List available versions +otdf-sdk-mgr versions list go --stable --latest 3 --table + +# Resolve version tags to SHAs +otdf-sdk-mgr versions resolve go main latest +``` + +### Other commands + +```bash +# Checkout SDK source +otdf-sdk-mgr checkout go main + +# Clean dist and source directories +otdf-sdk-mgr clean + +# Fix Java pom.xml after source checkout +otdf-sdk-mgr java-fixup +``` + +## How Release Installs Work + +- **Go**: Writes a `.version` file; `cli.sh`/`otdfctl.sh` use `go run github.com/opentdf/otdfctl@{version}` (no local compilation needed, Go caches the binary) +- **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` +- **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` + +## Source Builds + +Source builds (`tip` mode) delegate to `checkout-sdk-branch.sh` + `make`, which checks out source to `sdk/{lang}/src/` and compiles to `sdk/{lang}/dist/`. + +After changes to SDK source, rebuild: +```bash +otdf-sdk-mgr install tip go # or java, js + +# Or manually: checkout + make +cd sdk/go # or sdk/java, sdk/js +make + +# Verify build worked +ls -la dist/main/cli.sh +``` + +## Manual SDK Operations + +```bash +sdk/go/dist/main/cli.sh encrypt input.txt output.tdf --attr +sdk/go/dist/main/cli.sh decrypt output.tdf decrypted.txt +``` diff --git a/otdf-sdk-mgr/pyproject.toml b/otdf-sdk-mgr/pyproject.toml new file mode 100644 index 00000000..ce144e56 --- /dev/null +++ b/otdf-sdk-mgr/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "otdf-sdk-mgr" +version = "0.1.0" +description = "SDK artifact management CLI for OpenTDF cross-client tests" +requires-python = ">=3.11" +dependencies = [ + "gitpython>=3.1.46", + "rich>=13.7.0", + "typer>=0.12.0", +] + +[dependency-groups] +dev = [ + "pyright>=1.1.408", + "pytest>=8.0.0", + "ruff>=0.9.0", +] + +[project.scripts] +otdf-sdk-mgr = "otdf_sdk_mgr.cli:app" + +[build-system] +requires = ["uv_build>=0.9.28"] +build-backend = "uv_build" + +[tool.hatch.build.targets.wheel] +packages = ["src/otdf_sdk_mgr"] + +[tool.ruff] +target-version = "py311" +line-length = 100 diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/__init__.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/__init__.py new file mode 100644 index 00000000..cd99ab09 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/__init__.py @@ -0,0 +1,3 @@ +"""SDK artifact management CLI for OpenTDF cross-client tests.""" + +__version__ = "0.1.0" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/__main__.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/__main__.py new file mode 100644 index 00000000..a61eb8cd --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/__main__.py @@ -0,0 +1,5 @@ +"""Allow running as `python -m otdf_sdk_mgr`.""" + +from otdf_sdk_mgr.cli import app + +app() diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py new file mode 100644 index 00000000..3fc7e5c4 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -0,0 +1,74 @@ +"""SDK source checkout using bare repos and worktrees.""" + +from __future__ import annotations + +import subprocess +import sys +from typing import Any + +from otdf_sdk_mgr.config import SDK_BARE_REPOS, SDK_DIRS, SDK_GIT_URLS + + +def _run(cmd: list[str], **kwargs: Any) -> None: + """Run a command, exiting on failure.""" + result = subprocess.run(cmd, **kwargs) + if result.returncode != 0: + print(f"Error: Command '{' '.join(cmd)}' failed.", file=sys.stderr) + sys.exit(result.returncode) + + +def checkout_sdk_branch(language: str, branch: str) -> None: + """Clone bare repo and create/update a worktree for the given branch. + + Python port of checkout-sdk-branch.sh. + """ + if language not in SDK_DIRS: + print( + f"Error: Unsupported language '{language}'. " + f"Supported values are: {', '.join(SDK_DIRS)}", + file=sys.stderr, + ) + sys.exit(1) + + sdk_dir = SDK_DIRS[language] + bare_repo_name = SDK_BARE_REPOS[language] + # Strip .git suffix to get the base URL for git clone + repo_url = SDK_GIT_URLS[language].removesuffix(".git") + + bare_repo_path = sdk_dir / "src" / bare_repo_name + local_name = branch.replace("/", "--") + if local_name.startswith("sdk--"): + local_name = local_name.removeprefix("sdk--") + worktree_path = sdk_dir / "src" / local_name + + if not bare_repo_path.exists(): + print(f"Cloning {repo_url} as a bare repository into {bare_repo_path}...") + _run(["git", "clone", "--bare", repo_url, str(bare_repo_path)]) + else: + print(f"Bare repository already exists at {bare_repo_path}. Fetching updates...") + _run(["git", f"--git-dir={bare_repo_path}", "fetch", "--all"]) + + if worktree_path.exists(): + print(f"Worktree for branch '{branch}' already exists at {worktree_path}. Updating...") + _run( + [ + "git", + f"--git-dir={bare_repo_path}", + f"--work-tree={worktree_path}", + "pull", + "origin", + branch, + ] + ) + else: + print(f"Setting up worktree for branch '{branch}' at {worktree_path}...") + _run( + [ + "git", + f"--git-dir={bare_repo_path}", + "worktree", + "add", + str(worktree_path), + branch, + ] + ) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py new file mode 100644 index 00000000..2da0edf6 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py @@ -0,0 +1,89 @@ +"""Main CLI application for otdf-sdk-mgr.""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import Annotated, Optional + +import typer + +from otdf_sdk_mgr.cli_install import install_app +from otdf_sdk_mgr.cli_versions import versions_app +from otdf_sdk_mgr.config import ALL_SDKS, SDK_DIRS + +app = typer.Typer( + name="otdf-sdk-mgr", + help="SDK artifact management CLI for OpenTDF cross-client tests.", + no_args_is_help=True, +) + +app.add_typer(install_app, name="install") +app.add_typer(versions_app, name="versions") + + +@app.command() +def checkout( + sdk: Annotated[ + Optional[str], + typer.Argument(help="SDK to checkout (go, js, java)"), + ] = None, + branch: Annotated[str, typer.Argument(help="Branch to checkout")] = "main", + all_sdks: Annotated[bool, typer.Option("--all", help="Checkout all SDKs")] = False, +) -> None: + """Clone bare repo and create/update worktree for an SDK branch.""" + from otdf_sdk_mgr.checkout import checkout_sdk_branch + + if all_sdks: + for s in ALL_SDKS: + checkout_sdk_branch(s, branch) + elif sdk: + checkout_sdk_branch(sdk, branch) + else: + typer.echo("Error: provide an SDK name or use --all", err=True) + raise typer.Exit(1) + + +@app.command() +def clean( + dist_only: Annotated[ + bool, typer.Option("--dist-only", help="Only remove dist directories") + ] = False, + src_only: Annotated[ + bool, typer.Option("--src-only", help="Only remove source worktrees") + ] = False, +) -> None: + """Remove dist directories and/or source worktrees.""" + remove_dist = not src_only + remove_src = not dist_only + + for sdk in ALL_SDKS: + sdk_dir = SDK_DIRS[sdk] + if remove_dist: + dist_dir = sdk_dir / "dist" + if dist_dir.exists(): + shutil.rmtree(dist_dir) + typer.echo(f"Removed {dist_dir}") + + if remove_src: + src_dir = sdk_dir / "src" + if src_dir.exists(): + for entry in sorted(src_dir.iterdir()): + if entry.name.endswith(".git"): + continue + if entry.is_dir(): + shutil.rmtree(entry) + typer.echo(f"Removed {entry}") + + +@app.command("java-fixup") +def java_fixup( + base_dir: Annotated[ + Optional[Path], + typer.Argument(help="Base directory for Java source trees"), + ] = None, +) -> None: + """Fix pom.xml platform.branch property in Java SDK source trees.""" + from otdf_sdk_mgr.java_fixup import post_checkout_java_fixup + + post_checkout_java_fixup(base_dir) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py new file mode 100644 index 00000000..7234e196 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py @@ -0,0 +1,80 @@ +"""Install subcommand group for otdf-sdk-mgr.""" + +from __future__ import annotations + +from typing import Annotated, Optional + +import typer + +from otdf_sdk_mgr.config import ALL_SDKS + +install_app = typer.Typer(help="Install SDK CLI artifacts from registries or source.") + + +@install_app.command() +def stable( + sdks: Annotated[ + Optional[list[str]], + typer.Argument(help="SDKs to install (default: all)"), + ] = None, +) -> None: + """Install latest stable releases for each SDK.""" + from otdf_sdk_mgr.installers import cmd_stable + + cmd_stable(sdks or ALL_SDKS) + + +@install_app.command() +def lts( + sdks: Annotated[ + Optional[list[str]], + typer.Argument(help="SDKs to install (default: all)"), + ] = None, +) -> None: + """Install LTS versions for each SDK.""" + from otdf_sdk_mgr.installers import cmd_lts + + cmd_lts(sdks or ALL_SDKS) + + +@install_app.command() +def tip( + sdks: Annotated[ + Optional[list[str]], + typer.Argument(help="SDKs to build from source (default: all)"), + ] = None, +) -> None: + """Source checkout + build from main.""" + from otdf_sdk_mgr.installers import cmd_tip + + cmd_tip(sdks or ALL_SDKS) + + +@install_app.command() +def release( + specs: Annotated[ + list[str], + typer.Argument(help="Version specs as SDK:VERSION (e.g., go:v0.24.0)"), + ], +) -> None: + """Install specific released versions.""" + from otdf_sdk_mgr.installers import cmd_release + + cmd_release(specs) + + +@install_app.command() +def artifact( + sdk: Annotated[str, typer.Option(help="SDK to install")] = "", + version: Annotated[str, typer.Option(help="Version to install")] = "", + dist_name: Annotated[ + Optional[str], typer.Option("--dist-name", help="Override dist directory name") + ] = None, +) -> None: + """Install a single SDK version (used by CI).""" + if not sdk or not version: + typer.echo("Error: --sdk and --version are required", err=True) + raise typer.Exit(1) + from otdf_sdk_mgr.installers import cmd_install + + cmd_install(sdk, version, dist_name=dist_name) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py new file mode 100644 index 00000000..19a614ae --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -0,0 +1,129 @@ +"""Versions subcommand group for otdf-sdk-mgr.""" + +from __future__ import annotations + +import json +from typing import Annotated, Any, Optional + +import typer +from rich.console import Console +from rich.table import Table + +from otdf_sdk_mgr.config import SDK_GIT_URLS, SDK_TAG_INFIXES +from otdf_sdk_mgr.resolve import ResolveResult + +versions_app = typer.Typer(help="Query SDK version registries.") + + +@versions_app.command("list") +def list_versions( + sdk: Annotated[ + str, + typer.Argument(help="SDK to query (go, js, java, all)"), + ] = "all", + stable: Annotated[bool, typer.Option("--stable", help="Only stable versions")] = False, + latest: Annotated[ + Optional[int], typer.Option("--latest", help="Show only N most recent versions") + ] = None, + releases: Annotated[ + bool, typer.Option("--releases", help="Include GitHub Releases info for Java") + ] = False, + output_json: Annotated[bool, typer.Option("--json", help="JSON output (default)")] = False, + output_table: Annotated[ + bool, typer.Option("--table", help="Human-readable Rich table output") + ] = False, +) -> None: + """List available released versions of SDK CLIs.""" + from otdf_sdk_mgr.registry import ( + apply_filters, + list_go_versions, + list_java_github_releases, + list_java_maven_versions, + list_js_versions, + ) + + sdks = ["go", "js", "java"] if sdk == "all" else [sdk] + all_entries: list[dict[str, Any]] = [] + + for s in sdks: + if s == "go": + entries = list_go_versions() + all_entries.extend(apply_filters(entries, stable_only=stable, latest_n=latest)) + elif s == "js": + entries = list_js_versions() + all_entries.extend(apply_filters(entries, stable_only=stable, latest_n=latest)) + elif s == "java": + maven_entries = list_java_maven_versions() + all_entries.extend(apply_filters(maven_entries, stable_only=stable, latest_n=latest)) + if releases: + gh_entries = list_java_github_releases() + all_entries.extend(apply_filters(gh_entries, stable_only=stable, latest_n=latest)) + + if output_table: + _print_rich_table(all_entries) + else: + print(json.dumps(all_entries, indent=2)) + + +def _print_rich_table(entries: list[dict[str, Any]]) -> None: + """Print entries as a Rich table.""" + if not entries: + Console().print("[dim](no results)[/dim]") + return + + table = Table(show_header=True, header_style="bold") + table.add_column("SDK", width=6) + table.add_column("VERSION", width=20) + table.add_column("SOURCE", width=16) + table.add_column("STABLE", width=7) + table.add_column("HAS_CLI", width=8) + table.add_column("INSTALL_METHOD", min_width=40) + + for entry in entries: + stable_val = entry.get("stable", "") + stable_str = "yes" if stable_val else "no" if stable_val is not None else "" + has_cli_val = entry.get("has_cli", "") + has_cli_str = "yes" if has_cli_val else "no" if has_cli_val is not None else "" + table.add_row( + str(entry.get("sdk", "")), + str(entry.get("version", "")), + str(entry.get("source", "")), + stable_str, + has_cli_str, + str(entry.get("install_method", "")), + ) + + Console().print(table) + + +@versions_app.command("resolve") +def resolve_versions( + sdk: Annotated[str, typer.Argument(help="SDK to resolve (go, js, java, platform)")], + tags: Annotated[list[str], typer.Argument(help="Version tags to resolve")], +) -> None: + """Resolve version tags to git SHAs.""" + from otdf_sdk_mgr.resolve import ( + is_resolve_success, + lookup_additional_options, + resolve, + ) + + if sdk not in SDK_GIT_URLS: + typer.echo(f"Unknown SDK: {sdk}", err=True) + raise typer.Exit(2) + infix = SDK_TAG_INFIXES.get(sdk) + + results: list[ResolveResult] = [] + shas: set[str] = set() + for version in tags: + v = resolve(sdk, version, infix) + if is_resolve_success(v): + env = lookup_additional_options(sdk, v["tag"]) + if env: + v["env"] = env + if v["sha"] in shas: + continue + shas.add(v["sha"]) + results.append(v) + + print(json.dumps(results)) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py new file mode 100644 index 00000000..d754131c --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -0,0 +1,100 @@ +"""Constants and path discovery for SDK management.""" + +from __future__ import annotations + +from pathlib import Path + +# Discover the tests/ directory by walking up from this package +# In CI, the checkout may be named otdftests/ instead of tests/ +# Look for the xtest/sdk structure to identify the root +_PACKAGE_DIR = Path(__file__).resolve().parent +_TESTS_DIR = _PACKAGE_DIR +while _TESTS_DIR != _TESTS_DIR.parent: + if (_TESTS_DIR / "xtest" / "sdk").exists(): + break + _TESTS_DIR = _TESTS_DIR.parent + +SDK_DIR = _TESTS_DIR / "xtest" / "sdk" +if not SDK_DIR.exists(): + raise RuntimeError( + f"Could not locate xtest/sdk directory. " + f"Started from {_PACKAGE_DIR}, walked up to {_TESTS_DIR}. " + f"Expected to find xtest/sdk structure in repository root." + ) +GO_DIR = SDK_DIR / "go" +JS_DIR = SDK_DIR / "js" +JAVA_DIR = SDK_DIR / "java" +SCRIPTS_DIR = SDK_DIR / "scripts" + +SDK_DIRS: dict[str, Path] = { + "go": GO_DIR, + "js": JS_DIR, + "java": JAVA_DIR, +} + +# Git repository URLs (unified from resolve-version.py sdk_urls + list-versions.py sdk_git_urls) +SDK_GIT_URLS: dict[str, str] = { + "go": "https://github.com/opentdf/otdfctl.git", + "java": "https://github.com/opentdf/java-sdk.git", + "js": "https://github.com/opentdf/web-sdk.git", + "platform": "https://github.com/opentdf/platform.git", +} + +SDK_NPM_PACKAGES: dict[str, str] = { + "js": "@opentdf/ctl", +} + +SDK_MAVEN_COORDS: dict[str, dict[str, str]] = { + "java": { + "group": "io.opentdf.platform", + "artifact": "sdk", + "base_url": "https://repo1.maven.org/maven2/io/opentdf/platform/sdk", + }, +} + +SDK_GITHUB_REPOS: dict[str, str] = { + "java": "opentdf/java-sdk", +} + +GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl" + +LTS_VERSIONS: dict[str, str] = { + "go": "0.24.0", + "java": "0.9.0", + "js": "0.4.0", + "platform": "0.9.0", +} + +# Java SDK version -> compatible platform protocol branch +# Must stay in sync with resolve-version.py lookup_additional_options +JAVA_PLATFORM_BRANCH_MAP: dict[str, str] = { + "0.7.8": "protocol/go/v0.2.29", + "0.7.7": "protocol/go/v0.2.29", + "0.7.6": "protocol/go/v0.2.25", + "0.7.5": "protocol/go/v0.2.18", + "0.7.4": "protocol/go/v0.2.18", + "0.7.3": "protocol/go/v0.2.17", + "0.7.2": "protocol/go/v0.2.17", + "0.6.1": "protocol/go/v0.2.14", + "0.6.0": "protocol/go/v0.2.14", + "0.5.0": "protocol/go/v0.2.13", + "0.4.0": "protocol/go/v0.2.10", + "0.3.0": "protocol/go/v0.2.10", + "0.2.0": "protocol/go/v0.2.10", + "0.1.0": "protocol/go/v0.2.3", +} + +# Bare repo names per SDK (used by checkout) +SDK_BARE_REPOS: dict[str, str] = { + "go": "otdfctl.git", + "java": "java-sdk.git", + "js": "web-sdk.git", +} + +# Tag infixes for monorepo tag resolution +SDK_TAG_INFIXES: dict[str, str] = { + "js": "sdk", + "platform": "service", +} + +ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py new file mode 100644 index 00000000..326f48fb --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -0,0 +1,216 @@ +"""SDK CLI installation functions.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +from otdf_sdk_mgr.config import ( + GO_DIR, + JAVA_DIR, + JS_DIR, + LTS_VERSIONS, + SDK_DIRS, + SCRIPTS_DIR, +) +from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions +from otdf_sdk_mgr.semver import normalize_version + + +def install_go_release(version: str, dist_dir: Path) -> None: + """Install a Go CLI release by writing a .version file. + + The cli.sh and otdfctl.sh wrappers read .version and use + `go run github.com/opentdf/otdfctl@{version}` instead of a local binary. + """ + dist_dir.mkdir(parents=True, exist_ok=True) + tag = normalize_version(version) + (dist_dir / ".version").write_text(f"{tag}\n") + shutil.copy(GO_DIR / "cli.sh", dist_dir / "cli.sh") + shutil.copy(GO_DIR / "otdfctl.sh", dist_dir / "otdfctl.sh") + shutil.copy(GO_DIR / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") + print(f" Pre-warming Go cache for otdfctl@{tag}...") + result = subprocess.run( + ["go", "install", f"github.com/opentdf/otdfctl@{tag}"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print( + f" Warning: go install pre-warm failed (will retry at runtime): {result.stderr.strip()}" + ) + print(f" Go release {tag} installed to {dist_dir}") + + +def install_js_release(version: str, dist_dir: Path) -> None: + """Install a JS CLI release from npm registry.""" + dist_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(JS_DIR / "cli.sh", dist_dir / "cli.sh") + # Strip infix prefix (e.g., "sdk/v0.4.0" -> "v0.4.0") for npm install + v = version.split("/")[-1].lstrip("v") + print(f" Installing @opentdf/ctl@{v} from npm...") + subprocess.check_call( + ["npm", "install", f"@opentdf/ctl@{v}"], + cwd=dist_dir, + ) + print(f" JS release {v} installed to {dist_dir}") + + +def install_java_release(version: str, dist_dir: Path) -> None: + """Install a Java CLI release by downloading cmdline.jar from GitHub Releases. + + Falls back gracefully if the artifact is not available - the caller can + then build from source instead. + """ + tag = normalize_version(version) + url = f"https://github.com/opentdf/java-sdk/releases/download/{tag}/cmdline.jar" + + # Check if artifact exists before trying to download + try: + req = urllib.request.Request(url, method="HEAD") + urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as e: + if e.code == 404: + print(f" Warning: cmdline.jar not found for {tag}.", file=sys.stderr) + print(f" The release {tag} does not include a CLI artifact.", file=sys.stderr) + print(" This version will need to be built from source.", file=sys.stderr) + print( + f" Check: https://github.com/opentdf/java-sdk/releases/tag/{tag}", + file=sys.stderr, + ) + # Clean up partial dist dir if it was created + if dist_dir.exists(): + shutil.rmtree(dist_dir) + # Exit with error so caller knows to fall back to source build + sys.exit(1) + raise + except Exception as e: + print(f" Warning: Could not verify artifact availability: {e}", file=sys.stderr) + # Proceed with download attempt anyway + pass + + # Artifact exists, proceed with download + dist_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(JAVA_DIR / "cli.sh", dist_dir / "cli.sh") + jar_path = dist_dir / "cmdline.jar" + print(f" Downloading cmdline.jar from {url}...") + try: + urllib.request.urlretrieve(url, jar_path) + except urllib.error.HTTPError as e: + if e.code == 404: + print(f" Error: cmdline.jar not found for {tag} (race condition?).", file=sys.stderr) + sys.exit(1) + raise + print(f" Java release {tag} installed to {dist_dir}") + + +INSTALLERS = { + "go": install_go_release, + "js": install_js_release, + "java": install_java_release, +} + + +def install_release(sdk: str, version: str, dist_name: str | None = None) -> Path: + """Install a released version of an SDK CLI. + + Args: + sdk: One of "go", "js", "java" + version: Version string (e.g., "v0.24.0" or "0.24.0") + dist_name: Override the dist directory name (defaults to normalized version) + + Returns: + Path to the created dist directory + """ + if sdk not in INSTALLERS: + print( + f"Error: Unknown SDK '{sdk}'. Must be one of: {', '.join(INSTALLERS)}", + file=sys.stderr, + ) + sys.exit(1) + + name = dist_name or normalize_version(version) + dist_dir = SDK_DIRS[sdk] / "dist" / name + if dist_dir.exists(): + print(f" Dist directory already exists: {dist_dir} (skipping)") + return dist_dir + + INSTALLERS[sdk](version, dist_dir) + return dist_dir + + +def latest_stable_version(sdk: str) -> str | None: + """Find the latest stable version for an SDK that has a CLI available.""" + if sdk == "go": + versions = list_go_versions() + stable = [v for v in versions if v.get("stable", False)] + return stable[-1]["version"] if stable else None + elif sdk == "js": + versions = list_js_versions() + stable = [v for v in versions if v.get("stable", False)] + return stable[-1]["version"] if stable else None + elif sdk == "java": + releases = list_java_github_releases() + stable_with_cli = [ + v for v in releases if v.get("stable", False) and v.get("has_cli", False) + ] + return stable_with_cli[-1]["version"] if stable_with_cli else None + return None + + +def cmd_stable(sdks: list[str]) -> None: + """Install the latest stable release for each SDK.""" + for sdk in sdks: + print(f"Finding latest stable {sdk} release...") + version = latest_stable_version(sdk) + if version is None: + print(f" Warning: No stable version found for {sdk}, skipping") + continue + print(f" Latest stable {sdk}: {version}") + install_release(sdk, version) + + +def cmd_lts(sdks: list[str]) -> None: + """Install LTS versions for each SDK.""" + for sdk in sdks: + version = LTS_VERSIONS.get(sdk) + if version is None: + print(f" Warning: No LTS version defined for {sdk}, skipping") + continue + print(f"Installing LTS {sdk} {version}...") + install_release(sdk, version) + + +def cmd_tip(sdks: list[str]) -> None: + """Delegate to source checkout + make for head builds.""" + for sdk in sdks: + print(f"Checking out and building {sdk} from source...") + checkout_script = SCRIPTS_DIR / "checkout-sdk-branch.sh" + subprocess.check_call([str(checkout_script), sdk, "main"]) + make_dir = SDK_DIRS[sdk] + subprocess.check_call(["make"], cwd=make_dir) + print(f" {sdk} built from source") + + +def cmd_release(specs: list[str]) -> None: + """Install specific released versions from sdk:version specs.""" + for spec in specs: + if ":" not in spec: + print( + f"Error: Invalid spec '{spec}'. Use format sdk:version (e.g., go:v0.24.0)", + file=sys.stderr, + ) + sys.exit(1) + sdk, version = spec.split(":", 1) + print(f"Installing {sdk} {version} from registry...") + install_release(sdk, version) + + +def cmd_install(sdk: str, version: str, dist_name: str | None = None) -> None: + """Install a single SDK version (used by CI action).""" + print(f"Installing {sdk} {version}...") + install_release(sdk, version, dist_name=dist_name) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py new file mode 100644 index 00000000..e019aad3 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py @@ -0,0 +1,92 @@ +"""Post-checkout fixups for Java SDK source trees.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from otdf_sdk_mgr.config import JAVA_DIR, JAVA_PLATFORM_BRANCH_MAP + + +def _get_platform_branch(version: str) -> str: + """Map Java SDK version to compatible platform protocol branch.""" + return JAVA_PLATFORM_BRANCH_MAP.get(version, "main") + + +def post_checkout_java_fixup(base_dir: Path | None = None) -> None: + """Fix pom.xml platform.branch property in Java SDK source trees. + + Python port of post-checkout-java.sh. + """ + if base_dir is None: + base_dir = JAVA_DIR / "src" + + if not base_dir.exists(): + print(f"Base directory {base_dir} does not exist, nothing to fix.") + return + + for src_dir in sorted(base_dir.iterdir()): + if not src_dir.is_dir() or src_dir.name.endswith(".git"): + continue + + pom_file = src_dir / "sdk" / "pom.xml" + if not pom_file.exists(): + print(f"No pom.xml file found in {src_dir}, skipping.") + continue + + # Extract version from directory name (e.g., "v0.7.5" -> "0.7.5") + dir_name = src_dir.name + version = dir_name.lstrip("v") + platform_branch = _get_platform_branch(version) + + pom_content = pom_file.read_text() + + # Check if the correct platform.branch is already set + if f"{platform_branch}" in pom_content: + print(f"platform.branch already set to {platform_branch} in {pom_file}, skipping.") + continue + + # If we don't have a specific mapping (defaults to "main"), + # check if there's already a valid protocol/go branch set + if platform_branch == "main": + match = re.search(r"([^<]*)", pom_content) + if match and match.group(1).startswith("protocol/go/"): + print( + f"platform.branch already set to {match.group(1)} in {pom_file} " + f"(no mapping for version {version}), skipping." + ) + continue + + print(f"Updating {pom_file} (version={version}, platform.branch={platform_branch})...") + + if "" in pom_content: + # Replace existing platform.branch value + pom_content = re.sub( + r"[^<]*", + f"{platform_branch}", + pom_content, + ) + print(f"Updated existing platform.branch to {platform_branch} in {pom_file}") + elif "" in pom_content: + # Add the platform.branch property after + pom_content = pom_content.replace( + "", + f"\n {platform_branch}", + ) + # Replace hardcoded branch=main with branch=${platform.branch} + pom_content = pom_content.replace("branch=main", "branch=${platform.branch}") + print( + f"Added platform.branch={platform_branch} " + f"and updated branch references in {pom_file}" + ) + else: + # No section, directly replace branch=main + pom_content = pom_content.replace("branch=main", f"branch={platform_branch}") + print( + f"No section, directly replaced branch=main " + f"with branch={platform_branch} in {pom_file}" + ) + + pom_file.write_text(pom_content) + + print("Update complete.") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py new file mode 100644 index 00000000..009a2550 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -0,0 +1,178 @@ +"""Registry queries for SDK version discovery.""" + +from __future__ import annotations + +import json +import re +import sys +import urllib.error +import urllib.request +from typing import Any + +from otdf_sdk_mgr.config import ( + GO_INSTALL_PREFIX, + SDK_GITHUB_REPOS, + SDK_GIT_URLS, + SDK_MAVEN_COORDS, + SDK_NPM_PACKAGES, +) +from otdf_sdk_mgr.semver import is_stable, parse_semver, semver_sort_key + + +def fetch_json(url: str) -> Any: + """Fetch JSON from a URL.""" + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + + +def fetch_text(url: str) -> str: + """Fetch text content from a URL.""" + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode() + + +def list_go_versions() -> list[dict[str, Any]]: + """List Go SDK versions from git tags.""" + from git import Git + + repo = Git() + raw = repo.ls_remote(SDK_GIT_URLS["go"], tags=True) + results = [] + for line in raw.strip().split("\n"): + if not line: + continue + _, ref = line.split("\t", 1) + if ref.endswith("^{}"): + continue + tag = ref.removeprefix("refs/tags/") + if not parse_semver(tag): + continue + version = tag + results.append( + { + "sdk": "go", + "version": version, + "source": "git-tag", + "install_method": f"{GO_INSTALL_PREFIX}@{version}", + "stable": is_stable(version), + } + ) + results.sort(key=lambda r: semver_sort_key(r["version"])) + return results + + +def list_js_versions() -> list[dict[str, Any]]: + """List JS SDK versions from npm registry.""" + package = SDK_NPM_PACKAGES["js"] + url = f"https://registry.npmjs.org/{package}" + try: + data = fetch_json(url) + except urllib.error.URLError as e: + print(f"Warning: failed to fetch npm registry: {e}", file=sys.stderr) + return [] + + dist_tags: dict[str, str] = data.get("dist-tags", {}) + tag_lookup: dict[str, list[str]] = {} + for tag_name, tag_version in dist_tags.items(): + tag_lookup.setdefault(tag_version, []).append(tag_name) + + versions_dict: dict[str, Any] = data.get("versions", {}) + results = [] + for version in versions_dict: + if not parse_semver(version): + continue + entry: dict[str, Any] = { + "sdk": "js", + "version": version, + "source": "npm", + "install_method": f"npx {package}@{version}", + "stable": is_stable(version), + } + if version in tag_lookup: + entry["dist_tags"] = tag_lookup[version] + results.append(entry) + results.sort(key=lambda r: semver_sort_key(r["version"])) + return results + + +def list_java_maven_versions() -> list[dict[str, Any]]: + """List Java SDK versions from Maven Central metadata.""" + coords = SDK_MAVEN_COORDS["java"] + url = f"{coords['base_url']}/maven-metadata.xml" + try: + xml_text = fetch_text(url) + except urllib.error.URLError as e: + print(f"Warning: failed to fetch Maven metadata: {e}", file=sys.stderr) + return [] + + versions = re.findall(r"([^<]+)", xml_text) + results = [] + for version in versions: + if not parse_semver(version): + continue + results.append( + { + "sdk": "java", + "version": version, + "source": "maven", + "stable": is_stable(version), + "has_cli": False, + } + ) + results.sort(key=lambda r: semver_sort_key(r["version"])) + return results + + +def list_java_github_releases() -> list[dict[str, Any]]: + """List Java SDK versions from GitHub Releases (checks for cmdline.jar).""" + repo = SDK_GITHUB_REPOS["java"] + results = [] + page = 1 + while True: + url = f"https://api.github.com/repos/{repo}/releases?per_page=100&page={page}" + try: + releases = fetch_json(url) + except urllib.error.URLError as e: + print(f"Warning: failed to fetch GitHub releases: {e}", file=sys.stderr) + break + if not releases: + break + for release in releases: + tag = release.get("tag_name", "") + if not parse_semver(tag): + continue + assets = release.get("assets", []) + asset_names = [a["name"] for a in assets] + has_cli = any("cmdline" in name for name in asset_names) + entry: dict[str, Any] = { + "sdk": "java", + "version": tag, + "source": "github-release", + "stable": is_stable(tag) and not release.get("prerelease", False), + } + if asset_names: + entry["artifacts"] = asset_names + entry["has_cli"] = has_cli + if has_cli: + cli_asset = next(a for a in assets if "cmdline" in a["name"]) + entry["install_method"] = f"download from {cli_asset['browser_download_url']}" + results.append(entry) + page += 1 + results.sort(key=lambda r: semver_sort_key(r["version"])) + return results + + +def apply_filters( + entries: list[dict[str, Any]], + *, + stable_only: bool = False, + latest_n: int | None = None, +) -> list[dict[str, Any]]: + """Filter version entries by stability and count.""" + if stable_only: + entries = [e for e in entries if e.get("stable", False)] + if latest_n is not None: + entries = entries[-latest_n:] + return entries diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py new file mode 100644 index 00000000..7ad335bb --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -0,0 +1,301 @@ +"""Version resolution for SDK tags/branches/SHAs.""" + +from __future__ import annotations + +import json +import re +import sys +from typing import NotRequired, TypedDict, TypeGuard + +from git import Git + +from otdf_sdk_mgr.config import ( + JAVA_PLATFORM_BRANCH_MAP, + LTS_VERSIONS, + SDK_GIT_URLS, + SDK_TAG_INFIXES, +) + + +class ResolveSuccess(TypedDict): + sdk: str + alias: str + env: NotRequired[str] + head: NotRequired[bool] + pr: NotRequired[str] + release: NotRequired[str] + sha: str + tag: str + + +class ResolveError(TypedDict): + sdk: str + alias: str + err: str + + +ResolveResult = ResolveSuccess | ResolveError + + +def is_resolve_error(val: ResolveResult) -> TypeGuard[ResolveError]: + """Check if the given value is a ResolveError type.""" + return "err" in val + + +def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: + """Check if the given value is a ResolveSuccess type.""" + return "err" not in val and "sha" in val and "tag" in val + + +MERGE_QUEUE_REGEX = ( + r"^refs/heads/gh-readonly-queue/(?P[^/]+)/pr-(?P\d+)-(?P[a-f0-9]{40})$" +) + +SHA_REGEX = r"^[a-f0-9]{7,40}$" + + +def lookup_additional_options(sdk: str, version: str) -> str | None: + """Look up additional build options for a given SDK version.""" + if sdk != "java": + return None + if version.startswith("v"): + version = version[1:] + branch = JAVA_PLATFORM_BRANCH_MAP.get(version) + if branch: + return f"PLATFORM_BRANCH={branch}" + return None + + +def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: + """Resolve a version spec to a concrete SHA and tag.""" + sdk_url = SDK_GIT_URLS[sdk] + try: + repo = Git() + if version == "main" or version == "refs/heads/main": + all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] + sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": "main", + } + + if re.match(SHA_REGEX, version): + ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] + matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)] + if not matching_tags: + return { + "sdk": sdk, + "alias": version[:7], + "sha": version, + "tag": version, + } + if len(matching_tags) > 1: + for sha, tag in matching_tags: + if tag.startswith("refs/pull/"): + pr_number = tag.split("/")[2] + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": f"pull-{pr_number}", + } + for sha, tag in matching_tags: + mq_match = re.match(MERGE_QUEUE_REGEX, tag) + if mq_match: + to_branch = mq_match.group("branch") + pr_number = mq_match.group("pr_number") + if to_branch and pr_number: + return { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"mq-{to_branch}-{pr_number}", + } + suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] + flattag = "mq--" + suffix.replace("/", "--") + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": flattag, + } + head = False + if tag.startswith("refs/heads/"): + head = True + tag = tag.split("refs/heads/")[-1] + flattag = tag.replace("/", "--") + return { + "sdk": sdk, + "alias": version, + "head": head, + "sha": sha, + "tag": flattag, + } + + return { + "sdk": sdk, + "alias": version, + "err": ( + f"SHA {version} points to multiple tags, unable to differentiate: " + f"{', '.join(tag for _, tag in matching_tags)}" + ), + } + (sha, tag) = matching_tags[0] + if tag.startswith("refs/tags/"): + tag = tag.split("refs/tags/")[-1] + if infix: + tag = tag.split(f"{infix}/")[-1] + return { + "sdk": sdk, + "alias": version, + "sha": sha, + "tag": tag, + } + + if version.startswith("refs/pull/"): + merge_heads = [ + r.split("\t") for r in repo.ls_remote(sdk_url).split("\n") if r.endswith(version) + ] + pr_number = version.split("/")[2] + if not merge_heads: + return { + "sdk": sdk, + "alias": version, + "err": f"pull request {pr_number} not found in {sdk_url}", + } + sha, _ = merge_heads[0] + return { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"pull-{pr_number}", + } + + remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] + all_listed_tags = [ + (sha, tag.split("refs/tags/")[-1]) for (sha, tag) in remote_tags if "refs/tags/" in tag + ] + + all_listed_branches = { + tag.split("refs/heads/")[-1]: sha + for (sha, tag) in remote_tags + if tag.startswith("refs/heads/") + } + + if version in all_listed_branches: + sha = all_listed_branches[version] + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": version, + } + + if infix and version.startswith(f"{infix}/"): + version = version.split(f"{infix}/")[-1] + + listed_tags = all_listed_tags + if infix: + listed_tags = [ + (sha, tag.split(f"{infix}/")[-1]) + for (sha, tag) in listed_tags + if f"{infix}/" in tag + ] + semver_regex = r"v?\d+\.\d+\.\d+$" + listed_tags = [(sha, tag) for (sha, tag) in listed_tags if re.search(semver_regex, tag)] + listed_tags.sort(key=lambda item: list(map(int, item[1].strip("v").split(".")))) + alias = version + matching_tags = [] + if version == "latest": + # For Java, check if CLI artifacts are available and fall back to source build if not + if sdk == "java": + from otdf_sdk_mgr.registry import list_java_github_releases + + gh_releases = list_java_github_releases() + # Find the latest version with CLI artifact + versions_with_cli = [r for r in gh_releases if r.get("has_cli", False)] + if versions_with_cli: + # Use the latest version that has CLI + latest_with_cli_tag = versions_with_cli[-1]["version"] + matching_tags = [ + (sha, tag) + for (sha, tag) in listed_tags + if tag in [latest_with_cli_tag, latest_with_cli_tag.lstrip("v")] + ] + if not matching_tags: + # No versions with CLI found, fall back to building latest from source + sha, tag = listed_tags[-1] + return { + "sdk": sdk, + "alias": alias, + "head": True, # Mark as head to trigger source checkout + "sha": sha, + "tag": tag, + } + else: + matching_tags = listed_tags[-1:] + else: + if version == "lts": + version = LTS_VERSIONS[sdk] + matching_tags = [ + (sha, tag) for (sha, tag) in listed_tags if tag in [version, f"v{version}"] + ] + if not matching_tags: + raise ValueError(f"Tag [{version}] not found in [{sdk_url}]") + sha, tag = matching_tags[-1] + release = tag + if infix: + release = f"{infix}/{release}" + return { + "sdk": sdk, + "alias": alias, + "release": release, + "sha": sha, + "tag": tag, + } + except Exception as e: + return { + "sdk": sdk, + "alias": version, + "err": f"Error resolving version {version} for {sdk}: {e}", + } + + +def main() -> None: + """CLI entry point for backward-compatible resolve-version.py wrapper.""" + if len(sys.argv) < 3: + print("Usage: python resolve_version.py ", file=sys.stderr) + sys.exit(1) + + sdk = sys.argv[1] + versions = sys.argv[2:] + + if sdk not in SDK_GIT_URLS: + print(f"Unknown SDK: {sdk}", file=sys.stderr) + sys.exit(2) + infix = SDK_TAG_INFIXES.get(sdk) + + results: list[ResolveResult] = [] + shas: set[str] = set() + for version in versions: + v = resolve(sdk, version, infix) + if is_resolve_success(v): + env = lookup_additional_options(sdk, v["tag"]) + if env: + v["env"] = env + if v["sha"] in shas: + continue + shas.add(v["sha"]) + results.append(v) + + print(json.dumps(results)) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py new file mode 100644 index 00000000..74581a7c --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py @@ -0,0 +1,38 @@ +"""Semantic version parsing and utilities.""" + +from __future__ import annotations + +import re + +SEMVER_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-(.+))?$") + + +def parse_semver(version: str) -> tuple[int, int, int, str | None] | None: + """Parse a semver string into (major, minor, patch, pre) or None.""" + m = SEMVER_RE.match(version) + if not m: + return None + return int(m.group(1)), int(m.group(2)), int(m.group(3)), m.group(4) + + +def is_stable(version: str) -> bool: + """Return True if version has no pre-release suffix.""" + parsed = parse_semver(version) + if parsed is None: + return False + return parsed[3] is None + + +def semver_sort_key(version: str) -> tuple[int, int, int, int, str]: + """Sort key for semver strings. Stable versions sort after pre-release.""" + parsed = parse_semver(version) + if parsed is None: + return (0, 0, 0, 0, version) + major, minor, patch, pre = parsed + return (major, minor, patch, 0 if pre else 1, pre or "") + + +def normalize_version(version: str) -> str: + """Normalize version string to v-prefixed form.""" + v = version.strip().lstrip("v") + return f"v{v}" diff --git a/otdf-sdk-mgr/uv.lock b/otdf-sdk-mgr/uv.lock new file mode 100644 index 00000000..321f06c7 --- /dev/null +++ b/otdf-sdk-mgr/uv.lock @@ -0,0 +1,263 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.46" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/b5/59d16470a1f0dfe8c793f9ef56fd3826093fc52b3bd96d6b9d6c26c7e27b/gitpython-3.1.46.tar.gz", hash = "sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f", size = 215371, upload-time = "2026-01-01T15:37:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] +name = "otdf-sdk-mgr" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "gitpython" }, + { name = "rich" }, + { name = "typer" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pyright" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "gitpython", specifier = ">=3.1.46" }, + { name = "rich", specifier = ">=13.7.0" }, + { name = "typer", specifier = ">=0.12.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pyright", specifier = ">=1.1.408" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "ruff", specifier = ">=0.9.0" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyright" +version = "1.1.408" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/b2/5db700e52554b8f025faa9c3c624c59f1f6c8841ba81ab97641b54322f16/pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684", size = 4400578, upload-time = "2026-01-08T08:07:38.795Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/82/a2c93e32800940d9573fb28c346772a14778b84ba7524e691b324620ab89/pyright-1.1.408-py3-none-any.whl", hash = "sha256:090b32865f4fdb1e0e6cd82bf5618480d48eecd2eb2e70f960982a3d9a4c17c1", size = 6399144, upload-time = "2026-01-08T08:07:37.082Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "rich" +version = "14.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/99/a4cab2acbb884f80e558b0771e97e21e939c5dfb460f488d19df485e8298/rich-14.3.2.tar.gz", hash = "sha256:e712f11c1a562a11843306f5ed999475f09ac31ffb64281f73ab29ffdda8b3b8", size = 230143, upload-time = "2026-02-01T16:20:47.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl", hash = "sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69", size = 309963, upload-time = "2026-02-01T16:20:46.078Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/39/5cee96809fbca590abea6b46c6d1c586b49663d1d2830a751cc8fc42c666/ruff-0.15.0.tar.gz", hash = "sha256:6bdea47cdbea30d40f8f8d7d69c0854ba7c15420ec75a26f463290949d7f7e9a", size = 4524893, upload-time = "2026-02-03T17:53:35.357Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/88/3fd1b0aa4b6330d6aaa63a285bc96c9f71970351579152d231ed90914586/ruff-0.15.0-py3-none-linux_armv6l.whl", hash = "sha256:aac4ebaa612a82b23d45964586f24ae9bc23ca101919f5590bdb368d74ad5455", size = 10354332, upload-time = "2026-02-03T17:52:54.892Z" }, + { url = "https://files.pythonhosted.org/packages/72/f6/62e173fbb7eb75cc29fe2576a1e20f0a46f671a2587b5f604bfb0eaf5f6f/ruff-0.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dcd4be7cc75cfbbca24a98d04d0b9b36a270d0833241f776b788d59f4142b14d", size = 10767189, upload-time = "2026-02-03T17:53:19.778Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/968ae17b676d1d2ff101d56dc69cf333e3a4c985e1ec23803df84fc7bf9e/ruff-0.15.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d747e3319b2bce179c7c1eaad3d884dc0a199b5f4d5187620530adf9105268ce", size = 10075384, upload-time = "2026-02-03T17:53:29.241Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/9843c6044ab9e20af879c751487e61333ca79a2c8c3058b15722386b8cae/ruff-0.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:650bd9c56ae03102c51a5e4b554d74d825ff3abe4db22b90fd32d816c2e90621", size = 10481363, upload-time = "2026-02-03T17:52:43.332Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/4ada5ccf4cd1f532db1c8d44b6f664f2208d3d93acbeec18f82315e15193/ruff-0.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6664b7eac559e3048223a2da77769c2f92b43a6dfd4720cef42654299a599c9", size = 10187736, upload-time = "2026-02-03T17:53:00.522Z" }, + { url = "https://files.pythonhosted.org/packages/86/e2/f25eaecd446af7bb132af0a1d5b135a62971a41f5366ff41d06d25e77a91/ruff-0.15.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f811f97b0f092b35320d1556f3353bf238763420ade5d9e62ebd2b73f2ff179", size = 10968415, upload-time = "2026-02-03T17:53:15.705Z" }, + { url = "https://files.pythonhosted.org/packages/e7/dc/f06a8558d06333bf79b497d29a50c3a673d9251214e0d7ec78f90b30aa79/ruff-0.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761ec0a66680fab6454236635a39abaf14198818c8cdf691e036f4bc0f406b2d", size = 11809643, upload-time = "2026-02-03T17:53:23.031Z" }, + { url = "https://files.pythonhosted.org/packages/dd/45/0ece8db2c474ad7df13af3a6d50f76e22a09d078af63078f005057ca59eb/ruff-0.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:940f11c2604d317e797b289f4f9f3fa5555ffe4fb574b55ed006c3d9b6f0eb78", size = 11234787, upload-time = "2026-02-03T17:52:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/8a/d9/0e3a81467a120fd265658d127db648e4d3acfe3e4f6f5d4ea79fac47e587/ruff-0.15.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcbca3d40558789126da91d7ef9a7c87772ee107033db7191edefa34e2c7f1b4", size = 11112797, upload-time = "2026-02-03T17:52:49.274Z" }, + { url = "https://files.pythonhosted.org/packages/b2/cb/8c0b3b0c692683f8ff31351dfb6241047fa873a4481a76df4335a8bff716/ruff-0.15.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9a121a96db1d75fa3eb39c4539e607f628920dd72ff1f7c5ee4f1b768ac62d6e", size = 11033133, upload-time = "2026-02-03T17:53:33.105Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5e/23b87370cf0f9081a8c89a753e69a4e8778805b8802ccfe175cc410e50b9/ruff-0.15.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5298d518e493061f2eabd4abd067c7e4fb89e2f63291c94332e35631c07c3662", size = 10442646, upload-time = "2026-02-03T17:53:06.278Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9a/3c94de5ce642830167e6d00b5c75aacd73e6347b4c7fc6828699b150a5ee/ruff-0.15.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:afb6e603d6375ff0d6b0cee563fa21ab570fd15e65c852cb24922cef25050cf1", size = 10195750, upload-time = "2026-02-03T17:53:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/30/15/e396325080d600b436acc970848d69df9c13977942fb62bb8722d729bee8/ruff-0.15.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:77e515f6b15f828b94dc17d2b4ace334c9ddb7d9468c54b2f9ed2b9c1593ef16", size = 10676120, upload-time = "2026-02-03T17:53:09.363Z" }, + { url = "https://files.pythonhosted.org/packages/8d/c9/229a23d52a2983de1ad0fb0ee37d36e0257e6f28bfd6b498ee2c76361874/ruff-0.15.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6f6e80850a01eb13b3e42ee0ebdf6e4497151b48c35051aab51c101266d187a3", size = 11201636, upload-time = "2026-02-03T17:52:57.281Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b0/69adf22f4e24f3677208adb715c578266842e6e6a3cc77483f48dd999ede/ruff-0.15.0-py3-none-win32.whl", hash = "sha256:238a717ef803e501b6d51e0bdd0d2c6e8513fe9eec14002445134d3907cd46c3", size = 10465945, upload-time = "2026-02-03T17:53:12.591Z" }, + { url = "https://files.pythonhosted.org/packages/51/ad/f813b6e2c97e9b4598be25e94a9147b9af7e60523b0cb5d94d307c15229d/ruff-0.15.0-py3-none-win_amd64.whl", hash = "sha256:dd5e4d3301dc01de614da3cdffc33d4b1b96fb89e45721f1598e5532ccf78b18", size = 11564657, upload-time = "2026-02-03T17:52:51.893Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b0/2d823f6e77ebe560f4e397d078487e8d52c1516b331e3521bc75db4272ca/ruff-0.15.0-py3-none-win_arm64.whl", hash = "sha256:c480d632cc0ca3f0727acac8b7d053542d9e114a462a145d0b00e7cd658c515a", size = 10865753, upload-time = "2026-02-03T17:53:03.014Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, +] + +[[package]] +name = "typer" +version = "0.23.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/e6/44e073787aa57cd71c151f44855232feb0f748428fd5242d7366e3c4ae8b/typer-0.23.0.tar.gz", hash = "sha256:d8378833e47ada5d3d093fa20c4c63427cc4e27127f6b349a6c359463087d8cc", size = 120181, upload-time = "2026-02-11T15:22:18.637Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/ed/d6fca788b51d0d4640c4bc82d0e85bad4b49809bca36bf4af01b4dcb66a7/typer-0.23.0-py3-none-any.whl", hash = "sha256:79f4bc262b6c37872091072a3cb7cb6d7d79ee98c0c658b4364bdcde3c42c913", size = 56668, upload-time = "2026-02-11T15:22:21.075Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh index 755c84de..1d06da0e 100755 --- a/xtest/sdk/go/cli.sh +++ b/xtest/sdk/go/cli.sh @@ -22,7 +22,12 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then - cmd=(go run "github.com/opentdf/otdfctl@latest") + if [ -f "$SCRIPT_DIR/.version" ]; then + OTDFCTL_VERSION=$(tr -d '[:space:]' < "$SCRIPT_DIR/.version") + cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") + else + cmd=(go run "github.com/opentdf/otdfctl@latest") + fi fi if [ "$1" == "supports" ]; then diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh index 82c6dd50..f3a1a394 100755 --- a/xtest/sdk/go/otdfctl.sh +++ b/xtest/sdk/go/otdfctl.sh @@ -17,7 +17,12 @@ source "$XTEST_DIR/test.env" cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then - cmd=(go run github.com/opentdf/otdfctl@latest) + if [ -f "$SCRIPT_DIR/.version" ]; then + OTDFCTL_VERSION=$(tr -d '[:space:]' < "$SCRIPT_DIR/.version") + cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") + else + cmd=(go run "github.com/opentdf/otdfctl@latest") + fi fi cmd+=(--json) diff --git a/xtest/sdk/js/cli.sh b/xtest/sdk/js/cli.sh index 47af3099..ad3c3fa1 100755 --- a/xtest/sdk/js/cli.sh +++ b/xtest/sdk/js/cli.sh @@ -139,7 +139,7 @@ if [ -n "$XT_WITH_ASSERTIONS" ]; then assertions=$(realpath "$assertions") echo "Assertions are a file: $assertions" args+=(--assertions "$assertions") - elif [ "$(echo "$assertions" | jq -e . >/dev/null 2>&1 && echo valid || echo invalid)" == "valid" ]; then + elif [ "$(echo "$assertions" | jq -e . >/dev/null 2>&1 && echo valid || echo invalid)" == "valid" ]; then # Assertions are plain json echo "Assertions are plain json: $assertions" args+=(--assertions "$assertions") diff --git a/xtest/sdk/scripts/checkout-all.sh b/xtest/sdk/scripts/checkout-all.sh index c9cbbea9..ea644bd3 100755 --- a/xtest/sdk/scripts/checkout-all.sh +++ b/xtest/sdk/scripts/checkout-all.sh @@ -1,12 +1,13 @@ #!/bin/bash -# Checks out the latest `main` branch of each of the sdks under test -# and builds them. +# Backward-compatible wrapper. Use `otdf-sdk-mgr checkout --all` instead. -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" -for sdk in go java js; do - if ! "$SCRIPT_DIR/checkout-sdk-branch.sh" "$sdk" main; then - echo "Failed to checkout $sdk main branch" - exit 1 - fi -done +if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then + exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr checkout --all +else + for sdk in go java js; do + "$SCRIPT_DIR/checkout-sdk-branch.sh" "$sdk" main || exit 1 + done +fi diff --git a/xtest/sdk/scripts/checkout-sdk-branch.sh b/xtest/sdk/scripts/checkout-sdk-branch.sh index 55823145..ef662496 100755 --- a/xtest/sdk/scripts/checkout-sdk-branch.sh +++ b/xtest/sdk/scripts/checkout-sdk-branch.sh @@ -1,71 +1,20 @@ #!/bin/bash -# Refreshes to the latest sdk at branch in the appropriate folder. +# Backward-compatible wrapper. Use `otdf-sdk-mgr checkout` instead. # # Usage: ./checkout-sdk-branch.sh [sdk language] [branch] # Example: ./checkout-sdk-branch.sh js main -# Resolve script directory -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -XTEST_DIR=$(cd -- "$SCRIPT_DIR/../../" &>/dev/null && pwd) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" -# Parse arguments -LANGUAGE=${1:-js} -BRANCH=${2:-main} - -# Replace slashes in branch name with double dashes for local naming -LOCAL_NAME=${BRANCH//\//--} - -# Strip well known prefixes for monorepo output -if [[ $LOCAL_NAME == sdk--* ]]; then - LOCAL_NAME=${LOCAL_NAME#sdk--} -fi - -case "$LANGUAGE" in - js) - BARE_REPO_PATH="$XTEST_DIR/sdk/js/src/web-sdk.git" - WORKTREE_PATH="$XTEST_DIR/sdk/js/src/$LOCAL_NAME" - REPO_URL="https://github.com/opentdf/web-sdk" - ;; - java) - BARE_REPO_PATH="$XTEST_DIR/sdk/java/src/java-sdk.git" - WORKTREE_PATH="$XTEST_DIR/sdk/java/src/$LOCAL_NAME" - REPO_URL="https://github.com/opentdf/java-sdk" - ;; - go) - BARE_REPO_PATH="$XTEST_DIR/sdk/go/src/otdfctl.git" - WORKTREE_PATH="$XTEST_DIR/sdk/go/src/$LOCAL_NAME" - REPO_URL="https://github.com/opentdf/otdfctl" - ;; - *) - echo "Error: Unsupported language '$LANGUAGE'. Supported values are 'js', 'java', or 'go'." >&2 - exit 1 - ;; -esac - -# Function to execute a command and handle errors -run_command() { - "$@" - local status=$? - if [[ $status -ne 0 ]]; then - echo "Error: Command '$*' failed." >&2 - exit $status - fi -} - -# Clone the repository as bare if it doesn't exist -if [[ ! -d $BARE_REPO_PATH ]]; then - echo "Cloning $REPO_URL as a bare repository into $BARE_REPO_PATH..." - run_command git clone --bare "$REPO_URL" "$BARE_REPO_PATH" -else - echo "Bare repository already exists at $BARE_REPO_PATH. Fetching updates..." - run_command git --git-dir="$BARE_REPO_PATH" fetch --all -fi - -# Check if the worktree for the specified branch exists -if [[ -d $WORKTREE_PATH ]]; then - echo "Worktree for branch '$BRANCH' already exists at $WORKTREE_PATH. Updating..." - run_command git --git-dir="$BARE_REPO_PATH" --work-tree="$WORKTREE_PATH" pull origin "$BRANCH" +if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then + exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr checkout "${1:-js}" "${2:-main}" else - echo "Setting up worktree for branch '$BRANCH' at $WORKTREE_PATH..." - run_command git --git-dir="$BARE_REPO_PATH" worktree add "$WORKTREE_PATH" "$BRANCH" + # Fallback: direct Python import (works in CI without uv) + exec python3 -c " +import sys +sys.path.insert(0, '$PROJECT_DIR/src') +from otdf_sdk_mgr.checkout import checkout_sdk_branch +checkout_sdk_branch('${1:-js}', '${2:-main}') +" fi diff --git a/xtest/sdk/scripts/cleanup-all.sh b/xtest/sdk/scripts/cleanup-all.sh index 4614676e..fa126a3f 100755 --- a/xtest/sdk/scripts/cleanup-all.sh +++ b/xtest/sdk/scripts/cleanup-all.sh @@ -1,17 +1,22 @@ #!/bin/bash -# Removes the checked out branches of each of the sdks under test +# Backward-compatible wrapper. Use `otdf-sdk-mgr clean` instead. -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" -for sdk in go java js; do - rm -rf "$SCRIPT_DIR/../$sdk/dist" - for branch in "$SCRIPT_DIR/../${sdk}/src/"*; do - # Check if the path ends with .git - if [[ $branch == *.git ]]; then - continue - fi - if [ -d "$branch" ]; then - rm -rf "$branch" - fi +if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then + exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr clean +else + # Fallback: inline cleanup matching original behavior + for sdk in go java js; do + rm -rf "$SCRIPT_DIR/../$sdk/dist" + for branch in "$SCRIPT_DIR/../${sdk}/src/"*; do + if [[ $branch == *.git ]]; then + continue + fi + if [ -d "$branch" ]; then + rm -rf "$branch" + fi + done done -done +fi diff --git a/xtest/sdk/scripts/list-versions.py b/xtest/sdk/scripts/list-versions.py new file mode 100755 index 00000000..5eb5c998 --- /dev/null +++ b/xtest/sdk/scripts/list-versions.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Backward-compatible wrapper. Use `otdf-sdk-mgr versions list` instead. + +Also re-exports key functions for any code that imports this module directly. +""" + +import sys +from pathlib import Path + +# tests/otdf-sdk-mgr/src/ is three levels up from xtest/sdk/scripts/ +sys.path.insert( + 0, + str(Path(__file__).resolve().parent.parent.parent.parent / "otdf-sdk-mgr" / "src"), +) + +# Backward-compat: list-versions.py main() with argparse +import argparse # noqa: E402 +import json # noqa: E402 +from typing import Any # noqa: E402 + +from otdf_sdk_mgr.registry import ( # noqa: E402, F401 + apply_filters, + list_go_versions, + list_java_github_releases, + list_java_maven_versions, + list_js_versions, +) +from otdf_sdk_mgr.semver import ( # noqa: E402, F401 + is_stable, + parse_semver, + semver_sort_key, +) + + +def print_table(entries: list[dict[str, Any]]) -> None: + if not entries: + print("(no results)") + return + cols = { + "sdk": 6, + "version": 20, + "source": 16, + "stable": 7, + "has_cli": 8, + "install_method": 60, + } + header = " ".join(k.upper().ljust(v) for k, v in cols.items()) + print(header) + print("-" * len(header)) + for entry in entries: + row = [] + for key, width in cols.items(): + val = entry.get(key, "") + if isinstance(val, bool): + val = "yes" if val else "no" + row.append(str(val)[:width].ljust(width)) + print(" ".join(row)) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="List available released versions of OpenTDF SDK CLIs." + ) + parser.add_argument( + "sdk", + nargs="?", + default="all", + choices=["go", "js", "java", "all"], + help="SDK to query (default: all)", + ) + parser.add_argument( + "--stable", + action="store_true", + help="Only show stable (non-prerelease) versions", + ) + parser.add_argument( + "--latest", + type=int, + default=None, + metavar="N", + help="Show only the N most recent versions per source", + ) + parser.add_argument( + "--releases", + action="store_true", + help="Include GitHub Releases info for Java (slower)", + ) + output_group = parser.add_mutually_exclusive_group() + output_group.add_argument( + "--json", + action="store_true", + default=True, + dest="output_json", + help="JSON output (default)", + ) + output_group.add_argument( + "--table", + action="store_true", + help="Human-readable table output", + ) + args = parser.parse_args() + + sdks = ["go", "js", "java"] if args.sdk == "all" else [args.sdk] + all_entries: list[dict[str, Any]] = [] + + for sdk in sdks: + if sdk == "go": + entries = list_go_versions() + all_entries.extend( + apply_filters(entries, stable_only=args.stable, latest_n=args.latest) + ) + elif sdk == "js": + entries = list_js_versions() + all_entries.extend( + apply_filters(entries, stable_only=args.stable, latest_n=args.latest) + ) + elif sdk == "java": + maven_entries = list_java_maven_versions() + all_entries.extend( + apply_filters( + maven_entries, stable_only=args.stable, latest_n=args.latest + ) + ) + if args.releases: + gh_entries = list_java_github_releases() + all_entries.extend( + apply_filters( + gh_entries, stable_only=args.stable, latest_n=args.latest + ) + ) + + if args.table: + print_table(all_entries) + else: + print(json.dumps(all_entries, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/xtest/sdk/scripts/post-checkout-java.sh b/xtest/sdk/scripts/post-checkout-java.sh index 18a18d3b..8402e46e 100755 --- a/xtest/sdk/scripts/post-checkout-java.sh +++ b/xtest/sdk/scripts/post-checkout-java.sh @@ -1,93 +1,17 @@ #!/bin/bash +# Backward-compatible wrapper. Use `otdf-sdk-mgr java-fixup` instead. -# Post checkout cleanups for java -# Currently, this inserts the missing `platform.branch` property into the pom.xml files -# on older branches that do not have it defined. +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" -# Base directory for the script -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -BASE_DIR="$SCRIPT_DIR/../java/src" - -# Detect the operating system to use the correct sed syntax -if [[ "$(uname)" == "Darwin" ]]; then - SED_CMD="sed -i ''" +if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then + exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr java-fixup else - SED_CMD="sed -i" + # Fallback: direct Python import + exec python3 -c " +import sys +sys.path.insert(0, '$PROJECT_DIR/src') +from otdf_sdk_mgr.java_fixup import post_checkout_java_fixup +post_checkout_java_fixup() +" fi - -# Map Java SDK version to compatible platform protocol branch -# Must match the mappings in resolve-version.py -get_platform_branch() { - local version="$1" - case "$version" in - 0.7.8 | 0.7.7) echo "protocol/go/v0.2.29" ;; - 0.7.6) echo "protocol/go/v0.2.25" ;; - 0.7.5 | 0.7.4) echo "protocol/go/v0.2.18" ;; - 0.7.3 | 0.7.2) echo "protocol/go/v0.2.17" ;; - 0.6.1 | 0.6.0) echo "protocol/go/v0.2.14" ;; - 0.5.0) echo "protocol/go/v0.2.13" ;; - 0.4.0 | 0.3.0 | 0.2.0) echo "protocol/go/v0.2.10" ;; - 0.1.0) echo "protocol/go/v0.2.3" ;; - *) echo "main" ;; # Default to main for unknown/newer versions - esac - return 0 -} - -# Loop through all subdirectories in the base directory -find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type d -not -name "*.git" | while read -r SRC_DIR; do - POM_FILE="$SRC_DIR/sdk/pom.xml" - - # Skip if path or file does not exist - if [[ ! -f $POM_FILE ]]; then - echo "No pom.xml file found in $SRC_DIR, skipping." - continue - fi - - # Extract version from directory name (e.g., "v0.7.5" -> "0.7.5", "main" -> "main") - DIR_NAME=$(basename "$SRC_DIR") - VERSION="${DIR_NAME#v}" # Remove leading 'v' if present - PLATFORM_BRANCH=$(get_platform_branch "$VERSION") - - # Check if the correct platform.branch is already set - if grep -q "$PLATFORM_BRANCH" "$POM_FILE"; then - echo "platform.branch already set to $PLATFORM_BRANCH in $POM_FILE, skipping." - continue - fi - - # If we don't have a specific mapping for this version (defaults to "main"), - # check if the pom.xml already has a valid protocol/go branch set - don't overwrite it - if [[ "$PLATFORM_BRANCH" == "main" ]]; then - if grep -q "protocol/go/" "$POM_FILE"; then - EXISTING_BRANCH=$(grep -o "[^<]*" "$POM_FILE" | sed 's/<[^>]*>//g') - echo "platform.branch already set to $EXISTING_BRANCH in $POM_FILE (no mapping for version $VERSION), skipping." - continue - fi - fi - - echo "Updating $POM_FILE (version=$VERSION, platform.branch=$PLATFORM_BRANCH)..." - - # Check if platform.branch property exists (possibly with wrong value) - if grep -q "" "$POM_FILE"; then - # Replace existing platform.branch value with the correct one - $SED_CMD "s|[^<]*|$PLATFORM_BRANCH|g" "$POM_FILE" - echo "Updated existing platform.branch to $PLATFORM_BRANCH in $POM_FILE" - else - # Add the platform.branch property to the section - $SED_CMD "//a \\ - $PLATFORM_BRANCH" "$POM_FILE" - - # Only replace branch=main if the property now exists (sed above may have failed silently if no section) - if grep -q "" "$POM_FILE"; then - # Replace hardcoded branch=main with branch=${platform.branch} in the maven-antrun-plugin configuration - # shellcheck disable=SC2016 # Literal $; it is for a variable expansion in the maven file - $SED_CMD 's/branch=main/branch=${platform.branch}/g' "$POM_FILE" - echo "Added platform.branch=$PLATFORM_BRANCH and updated branch references in $POM_FILE" - else - # No section exists, directly replace branch=main with the actual branch value - $SED_CMD "s|branch=main|branch=$PLATFORM_BRANCH|g" "$POM_FILE" - echo "No section, directly replaced branch=main with branch=$PLATFORM_BRANCH in $POM_FILE" - fi - fi -done - -echo "Update complete." diff --git a/xtest/sdk/scripts/requirements.txt b/xtest/sdk/scripts/requirements.txt index 529b57ea..02905791 100644 --- a/xtest/sdk/scripts/requirements.txt +++ b/xtest/sdk/scripts/requirements.txt @@ -1 +1,3 @@ +# Minimal dependencies for backward-compatible wrappers. +# The canonical source is tests/otdf-sdk-mgr/pyproject.toml. GitPython==3.1.46 diff --git a/xtest/sdk/scripts/resolve-version.py b/xtest/sdk/scripts/resolve-version.py index ca6655c1..d380b9ca 100755 --- a/xtest/sdk/scripts/resolve-version.py +++ b/xtest/sdk/scripts/resolve-version.py @@ -1,374 +1,25 @@ #!/usr/bin/env python3 -# Use: python3 resolve-version.py -# -# Tag can be: -# main: the main branch -# latest: the latest release of the app (last tag) -# lts: one of a list of hard-coded 'supported' versions -# : a git SHA -# v0.1.2: a git tag that is a semantic version -# refs/pull/1234: a pull request ref -# -# The script will resolve the tags to their git SHAs and return it and other metadata in a JSON formatted list of objects. -# Fields of the object will be: -# sdk: the SDK name -# alias: the tag that was requested -# head: true if the tag is a head of a live branch -# tag: the resolved tag or branch name, if found -# sha: the current git SHA of the tag -# err: an error message if the tag could not be resolved, or resolved to multiple items -# pr: if set, the pr number associated with the tag -# release: if set, the release page for the tag -# -# The script will also check for duplicate SHAs and remove them from the output. -# -# Sample Input: -# -# python3 resolve-version.py go 0.15.0 latest decaf01 unreleased-name -# -# Sample Output: -# ```json -# [ -# { -# "sdk": "go", -# "alias": "0.15.0", -# "env": "ADDITIONAL_OPTION=per build metadata", -# "release": "v0.15.0", -# "sha": "a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0", -# "tag": "v0.15.0" -# }, -# { -# "sdk": "go", -# "alias": "latest", -# "release": "v0.15.1", -# "sha": "c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0a1b2", -# "tag": "v0.15.1" -# }, -# { -# "sdk": "go", -# "alias": "decaf01", -# "head": true, -# "pr": "1234", -# "sha": "decaf016g7h8i9j0k1l2m3n4o5p6q7r8s9t0a1b2", -# "tag": "refs/pull/1234/head" -# }, -# { -# "sdk": "go", -# "err": "not found", -# "tag": "unreleased-name" -# } -# ] -# ``` +"""Backward-compatible wrapper. Use `otdf-sdk-mgr versions resolve` instead.""" -import json -import re import sys -from typing import NotRequired, TypedDict, TypeGuard -from urllib.parse import quote - -from git import Git - - -class ResolveSuccess(TypedDict): - sdk: str # The SDK name - alias: str # The tag that was requested - env: NotRequired[str] # Additional options for the SDK - head: NotRequired[bool] # True if the tag is a head of a live branch - pr: NotRequired[str] # The pull request number associated with the tag - release: NotRequired[str] # The release name for the tag - sha: str # The current git SHA of the tag - tag: str # The resolved tag name - - -class ResolveError(TypedDict): - sdk: str # The SDK name - alias: str # The tag that was requested - err: str # The error message - - -ResolveResult = ResolveSuccess | ResolveError - - -def is_resolve_error(val: ResolveResult) -> TypeGuard[ResolveError]: - """Check if the given value is a ResolveError type.""" - return "err" in val - - -def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: - """Check if the given value is a ResolveSuccess type.""" - return "err" not in val and "sha" in val and "tag" in val - - -sdk_urls = { - "go": "https://github.com/opentdf/otdfctl.git", - "java": "https://github.com/opentdf/java-sdk.git", - "js": "https://github.com/opentdf/web-sdk.git", - "platform": "https://github.com/opentdf/platform.git", -} - -lts_versions = { - "go": "0.24.0", - "java": "0.9.0", - "js": "0.4.0", - "platform": "0.9.0", -} - - -merge_queue_regex = r"^refs/heads/gh-readonly-queue/(?P[^/]+)/pr-(?P\d+)-(?P[a-f0-9]{40})$" - -sha_regex = r"^[a-f0-9]{7,40}$" - - -def lookup_additional_options(sdk: str, version: str) -> str | None: - if sdk != "java": - return None - if version.startswith("v"): - version = version[1:] - match version: - case "0.7.8" | "0.7.7": - return "PLATFORM_BRANCH=protocol/go/v0.2.29" - case "0.7.6": - return "PLATFORM_BRANCH=protocol/go/v0.2.25" - case "0.7.5" | "0.7.4": - return "PLATFORM_BRANCH=protocol/go/v0.2.18" - case "0.7.3" | "0.7.2": - return "PLATFORM_BRANCH=protocol/go/v0.2.17" - case "0.6.1" | "0.6.0": - return "PLATFORM_BRANCH=protocol/go/v0.2.14" - case "0.5.0": - return "PLATFORM_BRANCH=protocol/go/v0.2.13" - case "0.4.0" | "0.3.0" | "0.2.0": - return "PLATFORM_BRANCH=protocol/go/v0.2.10" - case "0.1.0": - return "PLATFORM_BRANCH=protocol/go/v0.2.3" - case _: - return None - - -def resolve(sdk: str, version: str, infix: None | str) -> ResolveResult: - sdk_url = sdk_urls[sdk] - try: - repo = Git() - if version == "main" or version == "refs/heads/main": - all_heads = [ - r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n") - ] - sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] - return { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": "main", - } - - if re.match(sha_regex, version): - ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] - matching_tags = [ - (sha, tag) for (sha, tag) in ls_remote if sha.startswith(version) - ] - if not matching_tags: - # Not a head; maybe another commit has pushed to this branch since the job started - return { - "sdk": sdk, - "alias": version[:7], - "sha": version, - "tag": version, - } - if len(matching_tags) > 1: - # If multiple tags point to the same SHA, check for pull requests - # and return the first one. - for sha, tag in matching_tags: - if tag.startswith("refs/pull/"): - pr_number = tag.split("/")[2] - return { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": f"pull-{pr_number}", - } - # No pull request, probably a feature branch or release branch - for sha, tag in matching_tags: - mq_match = re.match(merge_queue_regex, tag) - if mq_match: - to_branch = mq_match.group("branch") - pr_number = mq_match.group("pr_number") - if to_branch and pr_number: - return { - "sdk": sdk, - "alias": version, - "head": True, - "pr": pr_number, - "sha": sha, - "tag": f"mq-{to_branch}-{pr_number}", - } - suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] - flattag = "mq--" + suffix.replace("/", "--") - return { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": flattag, - } - head = False - if tag.startswith("refs/heads/"): - head = True - tag = tag.split("refs/heads/")[-1] - flattag = tag.replace("/", "--") - return { - "sdk": sdk, - "alias": version, - "head": head, - "sha": sha, - "tag": flattag, - } - - return { - "sdk": sdk, - "alias": version, - "err": f"SHA {version} points to multiple tags, unable to differentiate: {', '.join(tag for _, tag in matching_tags)}", - } - (sha, tag) = matching_tags[0] - if tag.startswith("refs/tags/"): - tag = tag.split("refs/tags/")[-1] - if infix: - tag = tag.split(f"{infix}/")[-1] - return { - "sdk": sdk, - "alias": version, - "sha": sha, - "tag": tag, - } - - if version.startswith("refs/pull/"): - merge_heads = [ - r.split("\t") - for r in repo.ls_remote(sdk_url).split("\n") - if r.endswith(version) - ] - pr_number = version.split("/")[2] - if not merge_heads: - return { - "sdk": sdk, - "alias": version, - "err": f"pull request {pr_number} not found in {sdk_url}", - } - sha, _ = merge_heads[0] - return { - "sdk": sdk, - "alias": version, - "head": True, - "pr": pr_number, - "sha": sha, - "tag": f"pull-{pr_number}", - } - - remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] - all_listed_tags = [ - (sha, tag.split("refs/tags/")[-1]) - for (sha, tag) in remote_tags - if "refs/tags/" in tag - ] - - all_listed_branches = { - tag.split("refs/heads/")[-1]: sha - for (sha, tag) in remote_tags - if tag.startswith("refs/heads/") - } - - if version in all_listed_branches: - sha = all_listed_branches[version] - return { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": version, - } - - if infix and version.startswith(f"{infix}/"): - version = version.split(f"{infix}/")[-1] - - listed_tags = all_listed_tags - if infix: - listed_tags = [ - (sha, tag.split(f"{infix}/")[-1]) - for (sha, tag) in listed_tags - if f"{infix}/" in tag - ] - semver_regex = r"v?\d+\.\d+\.\d+$" - listed_tags = [ - (sha, tag) for (sha, tag) in listed_tags if re.search(semver_regex, tag) - ] - listed_tags.sort(key=lambda item: list(map(int, item[1].strip("v").split(".")))) - alias = version - matching_tags = [] - if version == "latest": - matching_tags = listed_tags[-1:] - else: - if version == "lts": - version = lts_versions[sdk] - matching_tags = [ - (sha, tag) - for (sha, tag) in listed_tags - if tag in [version, f"v{version}"] - ] - if not matching_tags: - raise ValueError(f"Tag [{version}] not found in [{sdk_url}]") - sha, tag = matching_tags[-1] - release = tag - if infix: - release = f"{infix}/{release}" - release = quote(release, safe="-_.~") - return { - "sdk": sdk, - "alias": alias, - "release": release, - "sha": sha, - "tag": tag, - } - except Exception as e: - return { - "sdk": sdk, - "alias": version, - "err": f"Error resolving version {version} for {sdk}: {e}", - } - - -def main(): - if len(sys.argv) < 3: - print("Usage: python resolve_version.py ", file=sys.stderr) - sys.exit(1) - - sdk = sys.argv[1] - versions = sys.argv[2:] - - if sdk not in sdk_urls: - print(f"Unknown SDK: {sdk}", file=sys.stderr) - sys.exit(2) - infix: None | str = None - if sdk == "js": - infix = "sdk" - if sdk == "platform": - infix = "service" - - results: list[ResolveResult] = [] - shas: set[str] = set() - for version in versions: - v = resolve(sdk, version, infix) - if is_resolve_success(v): - env = lookup_additional_options(sdk, v["tag"]) - if env: - v["env"] = env - if v["sha"] in shas: - continue - shas.add(v["sha"]) - results.append(v) - - print(json.dumps(results)) - +from pathlib import Path + +# tests/otdf-sdk-mgr/src/ is three levels up from xtest/sdk/scripts/ +sys.path.insert( + 0, + str(Path(__file__).resolve().parent.parent.parent.parent / "otdf-sdk-mgr" / "src"), +) + +# Re-export types and constants for any code that imports this module directly +from otdf_sdk_mgr.config import LTS_VERSIONS as lts_versions # noqa: E402, F401, N812 +from otdf_sdk_mgr.resolve import ( # noqa: E402, F401 + ResolveError, + ResolveResult, + ResolveSuccess, + is_resolve_error, + is_resolve_success, + main, +) if __name__ == "__main__": main() diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 24fa04e5..840842e3 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -76,9 +76,33 @@ runs: env: version_info: ${{ inputs.version-info }} + - name: install released versions + shell: bash + run: | + SDK_MGR_DIR="$(cd "${{ inputs.path }}/../.." && pwd)/otdf-sdk-mgr" + for row in $(echo "${version_info}" | jq -c '.[]'); do + tag=$(echo "$row" | jq -r '.tag') + head=$(echo "$row" | jq -r '.head // false') + release=$(echo "$row" | jq -r '.release // empty') + if [[ "$head" != "true" && -n "$release" ]]; then + echo "Installing ${{ inputs.sdk }} $tag from registry (release: $release)" + if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ + --sdk "${{ inputs.sdk }}" --version "$release" \ + --dist-name "$tag"; then + echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag" + echo " Will fall back to building from source" + echo "BUILD_FROM_SOURCE_$tag=true" >> "$GITHUB_ENV" + fi + fi + done + env: + version_info: ${{ inputs.version-info }} + - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - if: steps.resolve.outputs.version-a != '' + if: >- + steps.resolve.outputs.version-a != '' + && fromJson(steps.resolve.outputs.version-a).head == true with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }} persist-credentials: false @@ -87,7 +111,9 @@ runs: - name: checkout version b uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - if: steps.resolve.outputs.version-b != '' + if: >- + steps.resolve.outputs.version-b != '' + && fromJson(steps.resolve.outputs.version-b).head == true with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }} persist-credentials: false @@ -96,7 +122,9 @@ runs: - name: checkout version c uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - if: steps.resolve.outputs.version-c != '' + if: >- + steps.resolve.outputs.version-c != '' + && fromJson(steps.resolve.outputs.version-c).head == true with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }} persist-credentials: false @@ -105,7 +133,9 @@ runs: - name: checkout version d uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - if: steps.resolve.outputs.version-d != '' + if: >- + steps.resolve.outputs.version-d != '' + && fromJson(steps.resolve.outputs.version-d).head == true with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }} persist-credentials: false @@ -115,7 +145,8 @@ runs: - name: post checkout cleanups if: inputs.sdk == 'java' run: | - ${{ inputs.path }}/scripts/post-checkout-java.sh + SDK_MGR_DIR="$(cd "${{ inputs.path }}/../.." && pwd)/otdf-sdk-mgr" + uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr java-fixup shell: bash - name: save env from version-info @@ -131,4 +162,3 @@ runs: done env: version_info: ${{ inputs.version-info }} - From 20c407537da108a4f1f8998bf2a09404573d9b42 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Tue, 17 Feb 2026 23:33:20 -0500 Subject: [PATCH 02/24] cleanups --- xtest/sdk/go/cli.sh | 4 ++-- xtest/sdk/java/cli.sh | 2 +- xtest/sdk/js/cli.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh index 1d06da0e..fbfb987a 100755 --- a/xtest/sdk/go/cli.sh +++ b/xtest/sdk/go/cli.sh @@ -18,12 +18,12 @@ # XT_WITH_ATTRIBUTES [string] - Attributes to be used for encryption # XT_WITH_MIME_TYPE [string] - MIME type for the encrypted file # -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then if [ -f "$SCRIPT_DIR/.version" ]; then - OTDFCTL_VERSION=$(tr -d '[:space:]' < "$SCRIPT_DIR/.version") + OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") else cmd=(go run "github.com/opentdf/otdfctl@latest") diff --git a/xtest/sdk/java/cli.sh b/xtest/sdk/java/cli.sh index aae002f0..6457ed36 100755 --- a/xtest/sdk/java/cli.sh +++ b/xtest/sdk/java/cli.sh @@ -19,7 +19,7 @@ # XT_WITH_MIME_TYPE [string] - MIME type for the encrypted file # XT_WITH_TARGET_MODE [string] - Target spec mode for the encrypted file # -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) XTEST_DIR="$SCRIPT_DIR" while [ "$XTEST_DIR" != "/" ]; do diff --git a/xtest/sdk/js/cli.sh b/xtest/sdk/js/cli.sh index ad3c3fa1..0707e385 100755 --- a/xtest/sdk/js/cli.sh +++ b/xtest/sdk/js/cli.sh @@ -19,7 +19,7 @@ # XT_WITH_MIME_TYPE [string] - MIME type for the encrypted file # XT_WITH_TARGET_MODE [string] - Target spec mode for the encrypted file # -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) CTL=@opentdf/ctl if grep opentdf/cli "$SCRIPT_DIR/package.json"; then From 3baeb4477d23e931474aef4117fe4f001354ea5d Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Tue, 17 Feb 2026 23:39:11 -0500 Subject: [PATCH 03/24] Update otdfctl.sh --- xtest/sdk/go/otdfctl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh index f3a1a394..17fbb0c8 100755 --- a/xtest/sdk/go/otdfctl.sh +++ b/xtest/sdk/go/otdfctl.sh @@ -18,7 +18,7 @@ source "$XTEST_DIR/test.env" cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then if [ -f "$SCRIPT_DIR/.version" ]; then - OTDFCTL_VERSION=$(tr -d '[:space:]' < "$SCRIPT_DIR/.version") + OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") else cmd=(go run "github.com/opentdf/otdfctl@latest") From e1e8afba0162bb50194f63f3e1f48968767c37ac Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 00:14:32 -0500 Subject: [PATCH 04/24] remove old scripts --- otdf-sdk-mgr/README.md | 3 +- otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 5 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 4 +- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 5 +- otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py | 5 +- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 30 ----- xtest/README.md | 5 +- xtest/sdk/scripts/checkout-all.sh | 13 -- xtest/sdk/scripts/checkout-sdk-branch.sh | 20 --- xtest/sdk/scripts/cleanup-all.sh | 22 ---- xtest/sdk/scripts/list-versions.py | 139 -------------------- xtest/sdk/scripts/post-checkout-java.sh | 17 --- xtest/setup-cli-tool/action.yaml | 2 +- 13 files changed, 11 insertions(+), 259 deletions(-) delete mode 100755 xtest/sdk/scripts/checkout-all.sh delete mode 100755 xtest/sdk/scripts/checkout-sdk-branch.sh delete mode 100755 xtest/sdk/scripts/cleanup-all.sh delete mode 100755 xtest/sdk/scripts/list-versions.py delete mode 100755 xtest/sdk/scripts/post-checkout-java.sh diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index bdc7abee..6a33c69a 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -62,9 +62,10 @@ otdf-sdk-mgr java-fixup ## Source Builds -Source builds (`tip` mode) delegate to `checkout-sdk-branch.sh` + `make`, which checks out source to `sdk/{lang}/src/` and compiles to `sdk/{lang}/dist/`. +Source builds (`tip` mode) check out source to `sdk/{lang}/src/` and compile via `make` to `sdk/{lang}/dist/`. After changes to SDK source, rebuild: + ```bash otdf-sdk-mgr install tip go # or java, js diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py index 3fc7e5c4..31699151 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -18,10 +18,7 @@ def _run(cmd: list[str], **kwargs: Any) -> None: def checkout_sdk_branch(language: str, branch: str) -> None: - """Clone bare repo and create/update a worktree for the given branch. - - Python port of checkout-sdk-branch.sh. - """ + """Clone bare repo and create/update a worktree for the given branch.""" if language not in SDK_DIRS: print( f"Error: Unsupported language '{language}'. " diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index d754131c..592b283e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -32,7 +32,7 @@ "java": JAVA_DIR, } -# Git repository URLs (unified from resolve-version.py sdk_urls + list-versions.py sdk_git_urls) +# Git repository URLs SDK_GIT_URLS: dict[str, str] = { "go": "https://github.com/opentdf/otdfctl.git", "java": "https://github.com/opentdf/java-sdk.git", @@ -66,7 +66,7 @@ } # Java SDK version -> compatible platform protocol branch -# Must stay in sync with resolve-version.py lookup_additional_options +# Must stay in sync with otdf-sdk-mgr versions resolve's lookup_additional_options JAVA_PLATFORM_BRANCH_MAP: dict[str, str] = { "0.7.8": "protocol/go/v0.2.29", "0.7.7": "protocol/go/v0.2.29", diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 326f48fb..47ba3aab 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -15,8 +15,8 @@ JS_DIR, LTS_VERSIONS, SDK_DIRS, - SCRIPTS_DIR, ) +from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions from otdf_sdk_mgr.semver import normalize_version @@ -189,8 +189,7 @@ def cmd_tip(sdks: list[str]) -> None: """Delegate to source checkout + make for head builds.""" for sdk in sdks: print(f"Checking out and building {sdk} from source...") - checkout_script = SCRIPTS_DIR / "checkout-sdk-branch.sh" - subprocess.check_call([str(checkout_script), sdk, "main"]) + checkout_sdk_branch(sdk, "main") make_dir = SDK_DIRS[sdk] subprocess.check_call(["make"], cwd=make_dir) print(f" {sdk} built from source") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py index e019aad3..ad0a07bd 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py @@ -14,10 +14,7 @@ def _get_platform_branch(version: str) -> str: def post_checkout_java_fixup(base_dir: Path | None = None) -> None: - """Fix pom.xml platform.branch property in Java SDK source trees. - - Python port of post-checkout-java.sh. - """ + """Fix pom.xml platform.branch property in Java SDK source trees.""" if base_dir is None: base_dir = JAVA_DIR / "src" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 7ad335bb..ea44c50e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -269,33 +269,3 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: "alias": version, "err": f"Error resolving version {version} for {sdk}: {e}", } - - -def main() -> None: - """CLI entry point for backward-compatible resolve-version.py wrapper.""" - if len(sys.argv) < 3: - print("Usage: python resolve_version.py ", file=sys.stderr) - sys.exit(1) - - sdk = sys.argv[1] - versions = sys.argv[2:] - - if sdk not in SDK_GIT_URLS: - print(f"Unknown SDK: {sdk}", file=sys.stderr) - sys.exit(2) - infix = SDK_TAG_INFIXES.get(sdk) - - results: list[ResolveResult] = [] - shas: set[str] = set() - for version in versions: - v = resolve(sdk, version, infix) - if is_resolve_success(v): - env = lookup_additional_options(sdk, v["tag"]) - if env: - v["env"] = env - if v["sha"] in shas: - continue - shas.add(v["sha"]) - results.append(v) - - print(json.dumps(results)) diff --git a/xtest/README.md b/xtest/README.md index b36be0c9..6bdfcc40 100644 --- a/xtest/README.md +++ b/xtest/README.md @@ -26,16 +26,15 @@ This works by aliasing or checking out the source code for the different client To check out the current head versions of the sdks under test, run: ```sh - ./sdk/scripts/checkout-all.sh + uv run --project otdf-sdk-mgr otdf-sdk-mgr checkout --all ``` #### Download another tag of a specific sdk ```sh - ./sdk/scripts/checkout-sdk-branch.sh go v0.19.0 + uv run --project otdf-sdk-mgr otdf-sdk-mgr checkout go v0.19.0 ``` - #### Using locally checked out SDKs If you are developing a new feature or fix for a local SDK diff --git a/xtest/sdk/scripts/checkout-all.sh b/xtest/sdk/scripts/checkout-all.sh deleted file mode 100755 index ea644bd3..00000000 --- a/xtest/sdk/scripts/checkout-all.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Backward-compatible wrapper. Use `otdf-sdk-mgr checkout --all` instead. - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" - -if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then - exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr checkout --all -else - for sdk in go java js; do - "$SCRIPT_DIR/checkout-sdk-branch.sh" "$sdk" main || exit 1 - done -fi diff --git a/xtest/sdk/scripts/checkout-sdk-branch.sh b/xtest/sdk/scripts/checkout-sdk-branch.sh deleted file mode 100755 index ef662496..00000000 --- a/xtest/sdk/scripts/checkout-sdk-branch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Backward-compatible wrapper. Use `otdf-sdk-mgr checkout` instead. -# -# Usage: ./checkout-sdk-branch.sh [sdk language] [branch] -# Example: ./checkout-sdk-branch.sh js main - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" - -if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then - exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr checkout "${1:-js}" "${2:-main}" -else - # Fallback: direct Python import (works in CI without uv) - exec python3 -c " -import sys -sys.path.insert(0, '$PROJECT_DIR/src') -from otdf_sdk_mgr.checkout import checkout_sdk_branch -checkout_sdk_branch('${1:-js}', '${2:-main}') -" -fi diff --git a/xtest/sdk/scripts/cleanup-all.sh b/xtest/sdk/scripts/cleanup-all.sh deleted file mode 100755 index fa126a3f..00000000 --- a/xtest/sdk/scripts/cleanup-all.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Backward-compatible wrapper. Use `otdf-sdk-mgr clean` instead. - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" - -if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then - exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr clean -else - # Fallback: inline cleanup matching original behavior - for sdk in go java js; do - rm -rf "$SCRIPT_DIR/../$sdk/dist" - for branch in "$SCRIPT_DIR/../${sdk}/src/"*; do - if [[ $branch == *.git ]]; then - continue - fi - if [ -d "$branch" ]; then - rm -rf "$branch" - fi - done - done -fi diff --git a/xtest/sdk/scripts/list-versions.py b/xtest/sdk/scripts/list-versions.py deleted file mode 100755 index 5eb5c998..00000000 --- a/xtest/sdk/scripts/list-versions.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""Backward-compatible wrapper. Use `otdf-sdk-mgr versions list` instead. - -Also re-exports key functions for any code that imports this module directly. -""" - -import sys -from pathlib import Path - -# tests/otdf-sdk-mgr/src/ is three levels up from xtest/sdk/scripts/ -sys.path.insert( - 0, - str(Path(__file__).resolve().parent.parent.parent.parent / "otdf-sdk-mgr" / "src"), -) - -# Backward-compat: list-versions.py main() with argparse -import argparse # noqa: E402 -import json # noqa: E402 -from typing import Any # noqa: E402 - -from otdf_sdk_mgr.registry import ( # noqa: E402, F401 - apply_filters, - list_go_versions, - list_java_github_releases, - list_java_maven_versions, - list_js_versions, -) -from otdf_sdk_mgr.semver import ( # noqa: E402, F401 - is_stable, - parse_semver, - semver_sort_key, -) - - -def print_table(entries: list[dict[str, Any]]) -> None: - if not entries: - print("(no results)") - return - cols = { - "sdk": 6, - "version": 20, - "source": 16, - "stable": 7, - "has_cli": 8, - "install_method": 60, - } - header = " ".join(k.upper().ljust(v) for k, v in cols.items()) - print(header) - print("-" * len(header)) - for entry in entries: - row = [] - for key, width in cols.items(): - val = entry.get(key, "") - if isinstance(val, bool): - val = "yes" if val else "no" - row.append(str(val)[:width].ljust(width)) - print(" ".join(row)) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="List available released versions of OpenTDF SDK CLIs." - ) - parser.add_argument( - "sdk", - nargs="?", - default="all", - choices=["go", "js", "java", "all"], - help="SDK to query (default: all)", - ) - parser.add_argument( - "--stable", - action="store_true", - help="Only show stable (non-prerelease) versions", - ) - parser.add_argument( - "--latest", - type=int, - default=None, - metavar="N", - help="Show only the N most recent versions per source", - ) - parser.add_argument( - "--releases", - action="store_true", - help="Include GitHub Releases info for Java (slower)", - ) - output_group = parser.add_mutually_exclusive_group() - output_group.add_argument( - "--json", - action="store_true", - default=True, - dest="output_json", - help="JSON output (default)", - ) - output_group.add_argument( - "--table", - action="store_true", - help="Human-readable table output", - ) - args = parser.parse_args() - - sdks = ["go", "js", "java"] if args.sdk == "all" else [args.sdk] - all_entries: list[dict[str, Any]] = [] - - for sdk in sdks: - if sdk == "go": - entries = list_go_versions() - all_entries.extend( - apply_filters(entries, stable_only=args.stable, latest_n=args.latest) - ) - elif sdk == "js": - entries = list_js_versions() - all_entries.extend( - apply_filters(entries, stable_only=args.stable, latest_n=args.latest) - ) - elif sdk == "java": - maven_entries = list_java_maven_versions() - all_entries.extend( - apply_filters( - maven_entries, stable_only=args.stable, latest_n=args.latest - ) - ) - if args.releases: - gh_entries = list_java_github_releases() - all_entries.extend( - apply_filters( - gh_entries, stable_only=args.stable, latest_n=args.latest - ) - ) - - if args.table: - print_table(all_entries) - else: - print(json.dumps(all_entries, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/xtest/sdk/scripts/post-checkout-java.sh b/xtest/sdk/scripts/post-checkout-java.sh deleted file mode 100755 index 8402e46e..00000000 --- a/xtest/sdk/scripts/post-checkout-java.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Backward-compatible wrapper. Use `otdf-sdk-mgr java-fixup` instead. - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -PROJECT_DIR="$SCRIPT_DIR/../../../../otdf-sdk-mgr" - -if command -v uv &>/dev/null && [ -f "$PROJECT_DIR/pyproject.toml" ]; then - exec uv run --project "$PROJECT_DIR" otdf-sdk-mgr java-fixup -else - # Fallback: direct Python import - exec python3 -c " -import sys -sys.path.insert(0, '$PROJECT_DIR/src') -from otdf_sdk_mgr.java_fixup import post_checkout_java_fixup -post_checkout_java_fixup() -" -fi diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 840842e3..185410e0 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -6,7 +6,7 @@ inputs: sdk: description: The SDK to configure; one of go, java, js version-info: - description: JSON-encoded output of resolve-version.py + description: JSON-encoded output of otdf-sdk-mgr versions resolve required: true outputs: version-a: From c60a687c046206f5c22eddff055b10b5c5b90486 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 00:22:03 -0500 Subject: [PATCH 05/24] Update xtest.yml --- .github/workflows/xtest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 37086693..688564ae 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -132,7 +132,7 @@ jobs: const { execSync } = require('child_process'); const path = require('path'); - const sdkMgrDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk/otdf-sdk-mgr'); + const sdkMgrDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk-mgr'); const defaultTags = process.env.DEFAULT_TAGS || 'main'; core.setOutput('default-tags', defaultTags); From 9faa0d6ffaf12196ec906ecfc7bd36135f796874 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 00:33:16 -0500 Subject: [PATCH 06/24] Update xtest.yml --- .github/workflows/xtest.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 688564ae..882108e7 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -113,7 +113,9 @@ jobs: path: otdf-sdk persist-credentials: false repository: opentdf/tests - sparse-checkout: xtest/sdk + sparse-checkout: | + xtest/sdk + otdf-sdk-mgr - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b with: python-version: "3.14" From c5d2feb229fa20e0073c1047cceede4d9b823582 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 00:33:29 -0500 Subject: [PATCH 07/24] Update xtest.yml --- .github/workflows/xtest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 882108e7..f6ea1158 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -134,7 +134,7 @@ jobs: const { execSync } = require('child_process'); const path = require('path'); - const sdkMgrDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk-mgr'); + const sdkMgrDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk', 'otdf-sdk-mgr'); const defaultTags = process.env.DEFAULT_TAGS || 'main'; core.setOutput('default-tags', defaultTags); From d8d0a8838a29468a333176121bc4122acc247523 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 00:34:59 -0500 Subject: [PATCH 08/24] cleanups --- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index ea44c50e..34917425 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -2,9 +2,7 @@ from __future__ import annotations -import json import re -import sys from typing import NotRequired, TypedDict, TypeGuard from git import Git @@ -13,7 +11,6 @@ JAVA_PLATFORM_BRANCH_MAP, LTS_VERSIONS, SDK_GIT_URLS, - SDK_TAG_INFIXES, ) From d40d84e88d4892428c43f975cf6ea2ef1d80e663 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 09:01:26 -0500 Subject: [PATCH 09/24] chore(xtest): Removes dead code and comments to it --- AGENTS.md | 4 ++-- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/pyproject.toml | 3 --- xtest/sdk/scripts/requirements.txt | 3 --- xtest/sdk/scripts/resolve-version.py | 25 ------------------------- xtest/test_audit_logs_integration.py | 2 +- 6 files changed, 4 insertions(+), 35 deletions(-) delete mode 100644 xtest/sdk/scripts/requirements.txt delete mode 100755 xtest/sdk/scripts/resolve-version.py diff --git a/AGENTS.md b/AGENTS.md index 9165087a..46ac5c7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,10 +14,10 @@ This guide provides essential knowledge for AI agents performing updates, refact ### Configuring SDK Artifacts -Use `otdf-sdk-mgr` (uv-managed CLI in `tests/otdf-sdk-mgr/`) to install SDK CLIs from released artifacts or source. See `otdf-sdk-mgr/README.md` for full command reference. +Use `otdf-sdk-mgr` (uv-managed CLI in `otdf-sdk-mgr/`) to install SDK CLIs from released artifacts or source. See `otdf-sdk-mgr/README.md` for full command reference. ```bash -cd tests/otdf-sdk-mgr && uv tool install --editable . +cd otdf-sdk-mgr && uv tool install --editable . otdf-sdk-mgr install stable # Latest stable releases (recommended) otdf-sdk-mgr install tip go # Build from source ``` diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index 6a33c69a..ee2a2a8b 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -5,7 +5,7 @@ SDK artifact management CLI for OpenTDF cross-client tests. Installs SDK CLIs fr ## Installation ```bash -cd tests/otdf-sdk-mgr && uv tool install --editable . +cd otdf-sdk-mgr && uv tool install --editable . ``` ## Commands diff --git a/otdf-sdk-mgr/pyproject.toml b/otdf-sdk-mgr/pyproject.toml index ce144e56..6572cae6 100644 --- a/otdf-sdk-mgr/pyproject.toml +++ b/otdf-sdk-mgr/pyproject.toml @@ -23,9 +23,6 @@ otdf-sdk-mgr = "otdf_sdk_mgr.cli:app" requires = ["uv_build>=0.9.28"] build-backend = "uv_build" -[tool.hatch.build.targets.wheel] -packages = ["src/otdf_sdk_mgr"] - [tool.ruff] target-version = "py311" line-length = 100 diff --git a/xtest/sdk/scripts/requirements.txt b/xtest/sdk/scripts/requirements.txt deleted file mode 100644 index 02905791..00000000 --- a/xtest/sdk/scripts/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Minimal dependencies for backward-compatible wrappers. -# The canonical source is tests/otdf-sdk-mgr/pyproject.toml. -GitPython==3.1.46 diff --git a/xtest/sdk/scripts/resolve-version.py b/xtest/sdk/scripts/resolve-version.py deleted file mode 100755 index d380b9ca..00000000 --- a/xtest/sdk/scripts/resolve-version.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -"""Backward-compatible wrapper. Use `otdf-sdk-mgr versions resolve` instead.""" - -import sys -from pathlib import Path - -# tests/otdf-sdk-mgr/src/ is three levels up from xtest/sdk/scripts/ -sys.path.insert( - 0, - str(Path(__file__).resolve().parent.parent.parent.parent / "otdf-sdk-mgr" / "src"), -) - -# Re-export types and constants for any code that imports this module directly -from otdf_sdk_mgr.config import LTS_VERSIONS as lts_versions # noqa: E402, F401, N812 -from otdf_sdk_mgr.resolve import ( # noqa: E402, F401 - ResolveError, - ResolveResult, - ResolveSuccess, - is_resolve_error, - is_resolve_success, - main, -) - -if __name__ == "__main__": - main() diff --git a/xtest/test_audit_logs_integration.py b/xtest/test_audit_logs_integration.py index 16055202..6060d7b0 100644 --- a/xtest/test_audit_logs_integration.py +++ b/xtest/test_audit_logs_integration.py @@ -6,7 +6,7 @@ - Authorization decisions Run with: - cd tests/xtest + cd xtest uv run pytest test_audit_logs_integration.py --sdks go -v """ From dc32ae9f84d8b6863da112140bc0933153ee55de Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 09:28:13 -0500 Subject: [PATCH 10/24] fix(otdf-sdk-mgr): address PR #410 review issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - config: make SDK_DIR discovery lazy via get_sdk_dir()/get_sdk_dirs(); add OTDF_SDK_DIR env var override; no longer raises at import time - installers: replace sys.exit() with InstallError exception; fix partial-write by downloading to tempfile before moving into dist_dir; replace deprecated urlretrieve with urlopen + copyfileobj - resolve: guard LTS_VERSIONS lookup with explicit KeyError message - checkout: fix worktree update to use 'git -C pull' instead of broken --git-dir/--work-tree combination - cli_install: make --sdk and --version truly required (typer.Option(...) with no default); catch InstallError in release/artifact commands - registry: add GITHUB_TOKEN auth header support for GitHub API requests; warn with rate-limit reset time on 403/429 responses - action.yaml: sanitize tag names for env var use (dots→underscores); add 'determine source checkout needs' step that checks both head==true and BUILD_FROM_SOURCE_ fallback; update checkout step conditions Co-Authored-By: Claude Sonnet 4.6 --- otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 20 ++-- otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py | 5 +- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py | 23 +++-- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 68 ++++++++----- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 102 +++++++++++-------- otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py | 4 +- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 42 +++++++- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 5 + xtest/setup-cli-tool/action.yaml | 43 +++++++- 9 files changed, 208 insertions(+), 104 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py index 31699151..d9a0c994 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -6,7 +6,7 @@ import sys from typing import Any -from otdf_sdk_mgr.config import SDK_BARE_REPOS, SDK_DIRS, SDK_GIT_URLS +from otdf_sdk_mgr.config import SDK_BARE_REPOS, SDK_GIT_URLS, get_sdk_dirs def _run(cmd: list[str], **kwargs: Any) -> None: @@ -19,15 +19,16 @@ def _run(cmd: list[str], **kwargs: Any) -> None: def checkout_sdk_branch(language: str, branch: str) -> None: """Clone bare repo and create/update a worktree for the given branch.""" - if language not in SDK_DIRS: + sdk_dirs = get_sdk_dirs() + if language not in sdk_dirs: print( f"Error: Unsupported language '{language}'. " - f"Supported values are: {', '.join(SDK_DIRS)}", + f"Supported values are: {', '.join(sdk_dirs)}", file=sys.stderr, ) sys.exit(1) - sdk_dir = SDK_DIRS[language] + sdk_dir = sdk_dirs[language] bare_repo_name = SDK_BARE_REPOS[language] # Strip .git suffix to get the base URL for git clone repo_url = SDK_GIT_URLS[language].removesuffix(".git") @@ -47,16 +48,7 @@ def checkout_sdk_branch(language: str, branch: str) -> None: if worktree_path.exists(): print(f"Worktree for branch '{branch}' already exists at {worktree_path}. Updating...") - _run( - [ - "git", - f"--git-dir={bare_repo_path}", - f"--work-tree={worktree_path}", - "pull", - "origin", - branch, - ] - ) + _run(["git", "-C", str(worktree_path), "pull", "origin", branch]) else: print(f"Setting up worktree for branch '{branch}' at {worktree_path}...") _run( diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py index 2da0edf6..13089de9 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py @@ -10,7 +10,7 @@ from otdf_sdk_mgr.cli_install import install_app from otdf_sdk_mgr.cli_versions import versions_app -from otdf_sdk_mgr.config import ALL_SDKS, SDK_DIRS +from otdf_sdk_mgr.config import ALL_SDKS, get_sdk_dirs app = typer.Typer( name="otdf-sdk-mgr", @@ -57,8 +57,9 @@ def clean( remove_dist = not src_only remove_src = not dist_only + sdk_dirs = get_sdk_dirs() for sdk in ALL_SDKS: - sdk_dir = SDK_DIRS[sdk] + sdk_dir = sdk_dirs[sdk] if remove_dist: dist_dir = sdk_dir / "dist" if dist_dir.exists(): diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py index 7234e196..e3950d71 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py @@ -58,23 +58,28 @@ def release( ], ) -> None: """Install specific released versions.""" - from otdf_sdk_mgr.installers import cmd_release + from otdf_sdk_mgr.installers import InstallError, cmd_release - cmd_release(specs) + try: + cmd_release(specs) + except InstallError as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) @install_app.command() def artifact( - sdk: Annotated[str, typer.Option(help="SDK to install")] = "", - version: Annotated[str, typer.Option(help="Version to install")] = "", + sdk: Annotated[str, typer.Option(help="SDK to install")], + version: Annotated[str, typer.Option(help="Version to install")], dist_name: Annotated[ Optional[str], typer.Option("--dist-name", help="Override dist directory name") ] = None, ) -> None: """Install a single SDK version (used by CI).""" - if not sdk or not version: - typer.echo("Error: --sdk and --version are required", err=True) - raise typer.Exit(1) - from otdf_sdk_mgr.installers import cmd_install + from otdf_sdk_mgr.installers import InstallError, cmd_install - cmd_install(sdk, version, dist_name=dist_name) + try: + cmd_install(sdk, version, dist_name=dist_name) + except InstallError as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 592b283e..adf6c8b1 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -2,35 +2,49 @@ from __future__ import annotations +import os from pathlib import Path -# Discover the tests/ directory by walking up from this package -# In CI, the checkout may be named otdftests/ instead of tests/ -# Look for the xtest/sdk structure to identify the root -_PACKAGE_DIR = Path(__file__).resolve().parent -_TESTS_DIR = _PACKAGE_DIR -while _TESTS_DIR != _TESTS_DIR.parent: - if (_TESTS_DIR / "xtest" / "sdk").exists(): - break - _TESTS_DIR = _TESTS_DIR.parent - -SDK_DIR = _TESTS_DIR / "xtest" / "sdk" -if not SDK_DIR.exists(): - raise RuntimeError( - f"Could not locate xtest/sdk directory. " - f"Started from {_PACKAGE_DIR}, walked up to {_TESTS_DIR}. " - f"Expected to find xtest/sdk structure in repository root." - ) -GO_DIR = SDK_DIR / "go" -JS_DIR = SDK_DIR / "js" -JAVA_DIR = SDK_DIR / "java" -SCRIPTS_DIR = SDK_DIR / "scripts" - -SDK_DIRS: dict[str, Path] = { - "go": GO_DIR, - "js": JS_DIR, - "java": JAVA_DIR, -} + +def _find_sdk_dir() -> Path | None: + """Walk up from this package directory to find xtest/sdk.""" + current = Path(__file__).resolve().parent + while current != current.parent: + if (current / "xtest" / "sdk").exists(): + return current / "xtest" / "sdk" + current = current.parent + return None + + +def get_sdk_dir() -> Path: + """Return the SDK directory. + + Checks OTDF_SDK_DIR env var first, then walks up the directory tree + to find xtest/sdk. Raises RuntimeError if not found. + """ + env_dir = os.environ.get("OTDF_SDK_DIR") + if env_dir: + return Path(env_dir) + found = _find_sdk_dir() + if found is None: + pkg_dir = Path(__file__).resolve().parent + raise RuntimeError( + f"Could not locate xtest/sdk directory. " + f"Started from {pkg_dir}, walked up to filesystem root. " + f"Set OTDF_SDK_DIR env var to override." + ) + return found + + +def get_sdk_dirs() -> dict[str, Path]: + """Return per-SDK directories keyed by SDK name.""" + sdk_dir = get_sdk_dir() + return { + "go": sdk_dir / "go", + "js": sdk_dir / "js", + "java": sdk_dir / "java", + } + # Git repository URLs SDK_GIT_URLS: dict[str, str] = { diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 47ba3aab..5ae7c780 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -5,34 +5,38 @@ import shutil import subprocess import sys +import tempfile import urllib.error import urllib.request from pathlib import Path from otdf_sdk_mgr.config import ( - GO_DIR, - JAVA_DIR, - JS_DIR, LTS_VERSIONS, - SDK_DIRS, + get_sdk_dir, + get_sdk_dirs, ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions from otdf_sdk_mgr.semver import normalize_version +class InstallError(Exception): + """Raised when SDK installation fails.""" + + def install_go_release(version: str, dist_dir: Path) -> None: """Install a Go CLI release by writing a .version file. The cli.sh and otdfctl.sh wrappers read .version and use `go run github.com/opentdf/otdfctl@{version}` instead of a local binary. """ + go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) tag = normalize_version(version) (dist_dir / ".version").write_text(f"{tag}\n") - shutil.copy(GO_DIR / "cli.sh", dist_dir / "cli.sh") - shutil.copy(GO_DIR / "otdfctl.sh", dist_dir / "otdfctl.sh") - shutil.copy(GO_DIR / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") + shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") + shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh") + shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") print(f" Pre-warming Go cache for otdfctl@{tag}...") result = subprocess.run( ["go", "install", f"github.com/opentdf/otdfctl@{tag}"], @@ -48,8 +52,9 @@ def install_go_release(version: str, dist_dir: Path) -> None: def install_js_release(version: str, dist_dir: Path) -> None: """Install a JS CLI release from npm registry.""" + js_dir = get_sdk_dir() / "js" dist_dir.mkdir(parents=True, exist_ok=True) - shutil.copy(JS_DIR / "cli.sh", dist_dir / "cli.sh") + shutil.copy(js_dir / "cli.sh", dist_dir / "cli.sh") # Strip infix prefix (e.g., "sdk/v0.4.0" -> "v0.4.0") for npm install v = version.split("/")[-1].lstrip("v") print(f" Installing @opentdf/ctl@{v} from npm...") @@ -63,9 +68,10 @@ def install_js_release(version: str, dist_dir: Path) -> None: def install_java_release(version: str, dist_dir: Path) -> None: """Install a Java CLI release by downloading cmdline.jar from GitHub Releases. - Falls back gracefully if the artifact is not available - the caller can - then build from source instead. + Raises InstallError if the artifact is not available or download fails, + so the caller can fall back to building from source. """ + java_dir = get_sdk_dir() / "java" tag = normalize_version(version) url = f"https://github.com/opentdf/java-sdk/releases/download/{tag}/cmdline.jar" @@ -75,36 +81,47 @@ def install_java_release(version: str, dist_dir: Path) -> None: urllib.request.urlopen(req, timeout=10) except urllib.error.HTTPError as e: if e.code == 404: - print(f" Warning: cmdline.jar not found for {tag}.", file=sys.stderr) - print(f" The release {tag} does not include a CLI artifact.", file=sys.stderr) - print(" This version will need to be built from source.", file=sys.stderr) - print( - f" Check: https://github.com/opentdf/java-sdk/releases/tag/{tag}", - file=sys.stderr, + raise InstallError( + f"cmdline.jar not found for {tag}. " + f"The release {tag} does not include a CLI artifact. " + f"This version will need to be built from source. " + f"Check: https://github.com/opentdf/java-sdk/releases/tag/{tag}" ) - # Clean up partial dist dir if it was created - if dist_dir.exists(): - shutil.rmtree(dist_dir) - # Exit with error so caller knows to fall back to source build - sys.exit(1) raise except Exception as e: print(f" Warning: Could not verify artifact availability: {e}", file=sys.stderr) # Proceed with download attempt anyway - pass - # Artifact exists, proceed with download - dist_dir.mkdir(parents=True, exist_ok=True) - shutil.copy(JAVA_DIR / "cli.sh", dist_dir / "cli.sh") - jar_path = dist_dir / "cmdline.jar" - print(f" Downloading cmdline.jar from {url}...") + # Download to a temp file first to avoid partial writes + tmp_path: Path | None = None try: - urllib.request.urlretrieve(url, jar_path) - except urllib.error.HTTPError as e: - if e.code == 404: - print(f" Error: cmdline.jar not found for {tag} (race condition?).", file=sys.stderr) - sys.exit(1) + with tempfile.NamedTemporaryFile(delete=False, suffix=".jar") as tmp: + tmp_path = Path(tmp.name) + print(f" Downloading cmdline.jar from {url}...") + try: + with urllib.request.urlopen(url, timeout=60) as response: + with open(tmp_path, "wb") as f: + shutil.copyfileobj(response, f) + except urllib.error.HTTPError as e: + if e.code == 404: + raise InstallError( + f"cmdline.jar not found for {tag} (race condition?). " + f"Check: https://github.com/opentdf/java-sdk/releases/tag/{tag}" + ) + raise + + # Download succeeded — now create dist_dir and move files into place + dist_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(java_dir / "cli.sh", dist_dir / "cli.sh") + shutil.move(str(tmp_path), str(dist_dir / "cmdline.jar")) + tmp_path = None # Ownership transferred; don't clean up + except BaseException: + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + if dist_dir.exists() and not (dist_dir / "cmdline.jar").exists(): + shutil.rmtree(dist_dir, ignore_errors=True) raise + print(f" Java release {tag} installed to {dist_dir}") @@ -125,16 +142,18 @@ def install_release(sdk: str, version: str, dist_name: str | None = None) -> Pat Returns: Path to the created dist directory + + Raises: + InstallError: If the SDK is unknown or installation fails. """ if sdk not in INSTALLERS: - print( - f"Error: Unknown SDK '{sdk}'. Must be one of: {', '.join(INSTALLERS)}", - file=sys.stderr, + raise InstallError( + f"Unknown SDK '{sdk}'. Must be one of: {', '.join(INSTALLERS)}" ) - sys.exit(1) + sdk_dirs = get_sdk_dirs() name = dist_name or normalize_version(version) - dist_dir = SDK_DIRS[sdk] / "dist" / name + dist_dir = sdk_dirs[sdk] / "dist" / name if dist_dir.exists(): print(f" Dist directory already exists: {dist_dir} (skipping)") return dist_dir @@ -187,10 +206,11 @@ def cmd_lts(sdks: list[str]) -> None: def cmd_tip(sdks: list[str]) -> None: """Delegate to source checkout + make for head builds.""" + sdk_dirs = get_sdk_dirs() for sdk in sdks: print(f"Checking out and building {sdk} from source...") checkout_sdk_branch(sdk, "main") - make_dir = SDK_DIRS[sdk] + make_dir = sdk_dirs[sdk] subprocess.check_call(["make"], cwd=make_dir) print(f" {sdk} built from source") @@ -199,11 +219,9 @@ def cmd_release(specs: list[str]) -> None: """Install specific released versions from sdk:version specs.""" for spec in specs: if ":" not in spec: - print( - f"Error: Invalid spec '{spec}'. Use format sdk:version (e.g., go:v0.24.0)", - file=sys.stderr, + raise InstallError( + f"Invalid spec '{spec}'. Use format sdk:version (e.g., go:v0.24.0)" ) - sys.exit(1) sdk, version = spec.split(":", 1) print(f"Installing {sdk} {version} from registry...") install_release(sdk, version) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py index ad0a07bd..0712d05d 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py @@ -5,7 +5,7 @@ import re from pathlib import Path -from otdf_sdk_mgr.config import JAVA_DIR, JAVA_PLATFORM_BRANCH_MAP +from otdf_sdk_mgr.config import JAVA_PLATFORM_BRANCH_MAP, get_sdk_dir def _get_platform_branch(version: str) -> str: @@ -16,7 +16,7 @@ def _get_platform_branch(version: str) -> str: def post_checkout_java_fixup(base_dir: Path | None = None) -> None: """Fix pom.xml platform.branch property in Java SDK source trees.""" if base_dir is None: - base_dir = JAVA_DIR / "src" + base_dir = get_sdk_dir() / "java" / "src" if not base_dir.exists(): print(f"Base directory {base_dir} does not exist, nothing to fix.") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index 009a2550..e669189f 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -3,8 +3,10 @@ from __future__ import annotations import json +import os import re import sys +import time import urllib.error import urllib.request from typing import Any @@ -19,11 +21,45 @@ from otdf_sdk_mgr.semver import is_stable, parse_semver, semver_sort_key +def _github_headers() -> dict[str, str]: + """Return headers for GitHub API requests, including auth if GITHUB_TOKEN is set.""" + headers: dict[str, str] = {"Accept": "application/json"} + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + def fetch_json(url: str) -> Any: """Fetch JSON from a URL.""" - req = urllib.request.Request(url, headers={"Accept": "application/json"}) - with urllib.request.urlopen(req, timeout=30) as resp: - return json.loads(resp.read().decode()) + headers = {"Accept": "application/json"} + if url.startswith("https://api.github.com/"): + headers = _github_headers() + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + if e.code in (403, 429) and url.startswith("https://api.github.com/"): + reset_ts = e.headers.get("X-RateLimit-Reset") + if reset_ts: + reset_time = time.strftime( + "%Y-%m-%d %H:%M:%S UTC", time.gmtime(int(reset_ts)) + ) + print( + f"Warning: GitHub API rate limit exceeded. " + f"Rate limit resets at {reset_time}. " + f"Set GITHUB_TOKEN to increase rate limits.", + file=sys.stderr, + ) + else: + print( + f"Warning: GitHub API returned {e.code}. " + f"Set GITHUB_TOKEN to authenticate requests.", + file=sys.stderr, + ) + raise + raise def fetch_text(url: str) -> str: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 34917425..d6d03006 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -243,6 +243,11 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: matching_tags = listed_tags[-1:] else: if version == "lts": + if sdk not in LTS_VERSIONS: + raise ValueError( + f"No LTS version defined for SDK '{sdk}'. " + f"Add it to LTS_VERSIONS in config.py." + ) version = LTS_VERSIONS[sdk] matching_tags = [ (sha, tag) for (sha, tag) in listed_tags if tag in [version, f"v{version}"] diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 185410e0..75a9a1fa 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -86,23 +86,56 @@ runs: release=$(echo "$row" | jq -r '.release // empty') if [[ "$head" != "true" && -n "$release" ]]; then echo "Installing ${{ inputs.sdk }} $tag from registry (release: $release)" + # Sanitize tag for use as an env var name (replace . with _) + tag_sanitized="${tag//./_}" if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ --sdk "${{ inputs.sdk }}" --version "$release" \ --dist-name "$tag"; then echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag" echo " Will fall back to building from source" - echo "BUILD_FROM_SOURCE_$tag=true" >> "$GITHUB_ENV" + echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV" fi fi done env: version_info: ${{ inputs.version-info }} + - name: determine source checkout needs + id: check-source + shell: bash + run: | + # Determine which version slots need source checkout. + # A slot needs checkout if it is a head version OR if artifact install failed + # (BUILD_FROM_SOURCE_ was set in the previous step). + for slot in a b c d; do + case "$slot" in + a) row=$(echo "${version_info}" | jq -rc '.[0] // empty') ;; + b) row=$(echo "${version_info}" | jq -rc '.[1] // empty') ;; + c) row=$(echo "${version_info}" | jq -rc '.[2] // empty') ;; + d) row=$(echo "${version_info}" | jq -rc '.[3] // empty') ;; + esac + if [[ -z "$row" ]]; then + echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT" + continue + fi + tag=$(echo "$row" | jq -r '.tag') + head=$(echo "$row" | jq -r '.head // false') + tag_sanitized="${tag//./_}" + build_from_source_var="BUILD_FROM_SOURCE_${tag_sanitized}" + if [[ "$head" == "true" || "${!build_from_source_var}" == "true" ]]; then + echo "needs-source-${slot}=true" >> "$GITHUB_OUTPUT" + else + echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT" + fi + done + env: + version_info: ${{ inputs.version-info }} + - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- steps.resolve.outputs.version-a != '' - && fromJson(steps.resolve.outputs.version-a).head == true + && steps.check-source.outputs.needs-source-a == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }} persist-credentials: false @@ -113,7 +146,7 @@ runs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- steps.resolve.outputs.version-b != '' - && fromJson(steps.resolve.outputs.version-b).head == true + && steps.check-source.outputs.needs-source-b == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }} persist-credentials: false @@ -124,7 +157,7 @@ runs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- steps.resolve.outputs.version-c != '' - && fromJson(steps.resolve.outputs.version-c).head == true + && steps.check-source.outputs.needs-source-c == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }} persist-credentials: false @@ -135,7 +168,7 @@ runs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- steps.resolve.outputs.version-d != '' - && fromJson(steps.resolve.outputs.version-d).head == true + && steps.check-source.outputs.needs-source-d == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }} persist-credentials: false From a665d5ef3950ff1b489913d85202df3399fb0b7f Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 09:32:35 -0500 Subject: [PATCH 11/24] chore(xtest): fmt --- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 8 ++------ otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 5ae7c780..541a6b4b 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -147,9 +147,7 @@ def install_release(sdk: str, version: str, dist_name: str | None = None) -> Pat InstallError: If the SDK is unknown or installation fails. """ if sdk not in INSTALLERS: - raise InstallError( - f"Unknown SDK '{sdk}'. Must be one of: {', '.join(INSTALLERS)}" - ) + raise InstallError(f"Unknown SDK '{sdk}'. Must be one of: {', '.join(INSTALLERS)}") sdk_dirs = get_sdk_dirs() name = dist_name or normalize_version(version) @@ -219,9 +217,7 @@ def cmd_release(specs: list[str]) -> None: """Install specific released versions from sdk:version specs.""" for spec in specs: if ":" not in spec: - raise InstallError( - f"Invalid spec '{spec}'. Use format sdk:version (e.g., go:v0.24.0)" - ) + raise InstallError(f"Invalid spec '{spec}'. Use format sdk:version (e.g., go:v0.24.0)") sdk, version = spec.split(":", 1) print(f"Installing {sdk} {version} from registry...") install_release(sdk, version) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index e669189f..8f8dd34e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -43,9 +43,7 @@ def fetch_json(url: str) -> Any: if e.code in (403, 429) and url.startswith("https://api.github.com/"): reset_ts = e.headers.get("X-RateLimit-Reset") if reset_ts: - reset_time = time.strftime( - "%Y-%m-%d %H:%M:%S UTC", time.gmtime(int(reset_ts)) - ) + reset_time = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime(int(reset_ts))) print( f"Warning: GitHub API rate limit exceeded. " f"Rate limit resets at {reset_time}. " From 5ef4548192906fc64c1ba88e6a144d0505f2d2cb Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 11:54:02 -0500 Subject: [PATCH 12/24] chore(otdf-sdk-mgr): address PR #410 review issues (round 2) - Fix SonarCloud security hotspot: replace execSync with spawnSync in xtest.yml to avoid command injection via template literal interpolation - Use removeprefix("v") instead of lstrip("v") in java_fixup.py for correct single-prefix stripping semantics - Wrap urlopen HEAD check in context manager to prevent resource leak; narrow broad except Exception to (URLError, OSError) in installers.py - Replace sys.exit calls in checkout.py with proper exceptions (CalledProcessError, ValueError); handle them in CLI entry point - Remove redundant --json flag from cli_versions.py list command - Fix shell word-splitting in setup-cli-tool/action.yaml by piping through while read instead of for-in; improve tag_sanitized regex Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/xtest.yml | 8 +++++-- otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 14 +++++-------- otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py | 21 ++++++++++++------- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 1 - otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 5 +++-- otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py | 2 +- xtest/setup-cli-tool/action.yaml | 6 +++--- 7 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index f6ea1158..ab5134b8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -131,7 +131,7 @@ jobs: .replace(/"/g, """) .replace(/'/g, "'"); } - const { execSync } = require('child_process'); + const { spawnSync } = require('child_process'); const path = require('path'); const sdkMgrDir = path.join(process.env.GITHUB_WORKSPACE, 'otdf-sdk', 'otdf-sdk-mgr'); @@ -149,7 +149,11 @@ jobs: for (const [sdkType, ref] of Object.entries(refs)) { try { - const output = execSync(`uv run --project ${sdkMgrDir} otdf-sdk-mgr versions resolve ${sdkType} ${ref}`, { cwd: sdkMgrDir }).toString(); + const result = spawnSync('uv', ['run', '--project', sdkMgrDir, 'otdf-sdk-mgr', 'versions', 'resolve', sdkType, ref], { cwd: sdkMgrDir, encoding: 'utf-8' }); + if (result.status !== 0) { + throw new Error(result.stderr || `Process exited with code ${result.status}`); + } + const output = result.stdout; const ojson = JSON.parse(output); if (!!ojson.err) { throw new Error(ojson.err); diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py index d9a0c994..618e4629 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -3,30 +3,26 @@ from __future__ import annotations import subprocess -import sys from typing import Any from otdf_sdk_mgr.config import SDK_BARE_REPOS, SDK_GIT_URLS, get_sdk_dirs def _run(cmd: list[str], **kwargs: Any) -> None: - """Run a command, exiting on failure.""" + """Run a command, raising on failure.""" result = subprocess.run(cmd, **kwargs) if result.returncode != 0: - print(f"Error: Command '{' '.join(cmd)}' failed.", file=sys.stderr) - sys.exit(result.returncode) + raise subprocess.CalledProcessError(result.returncode, cmd) def checkout_sdk_branch(language: str, branch: str) -> None: """Clone bare repo and create/update a worktree for the given branch.""" sdk_dirs = get_sdk_dirs() if language not in sdk_dirs: - print( - f"Error: Unsupported language '{language}'. " - f"Supported values are: {', '.join(sdk_dirs)}", - file=sys.stderr, + raise ValueError( + f"Unsupported language '{language}'. " + f"Supported values are: {', '.join(sdk_dirs)}" ) - sys.exit(1) sdk_dir = sdk_dirs[language] bare_repo_name = SDK_BARE_REPOS[language] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py index 13089de9..24148bdd 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil +import subprocess from pathlib import Path from typing import Annotated, Optional @@ -34,14 +35,18 @@ def checkout( """Clone bare repo and create/update worktree for an SDK branch.""" from otdf_sdk_mgr.checkout import checkout_sdk_branch - if all_sdks: - for s in ALL_SDKS: - checkout_sdk_branch(s, branch) - elif sdk: - checkout_sdk_branch(sdk, branch) - else: - typer.echo("Error: provide an SDK name or use --all", err=True) - raise typer.Exit(1) + try: + if all_sdks: + for s in ALL_SDKS: + checkout_sdk_branch(s, branch) + elif sdk: + checkout_sdk_branch(sdk, branch) + else: + typer.echo("Error: provide an SDK name or use --all", err=True) + raise typer.Exit(1) + except (ValueError, subprocess.CalledProcessError) as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) from e @app.command() diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py index 19a614ae..19188b12 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -28,7 +28,6 @@ def list_versions( releases: Annotated[ bool, typer.Option("--releases", help="Include GitHub Releases info for Java") ] = False, - output_json: Annotated[bool, typer.Option("--json", help="JSON output (default)")] = False, output_table: Annotated[ bool, typer.Option("--table", help="Human-readable Rich table output") ] = False, diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 541a6b4b..e7c22ae0 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -78,7 +78,8 @@ def install_java_release(version: str, dist_dir: Path) -> None: # Check if artifact exists before trying to download try: req = urllib.request.Request(url, method="HEAD") - urllib.request.urlopen(req, timeout=10) + with urllib.request.urlopen(req, timeout=10): + pass except urllib.error.HTTPError as e: if e.code == 404: raise InstallError( @@ -88,7 +89,7 @@ def install_java_release(version: str, dist_dir: Path) -> None: f"Check: https://github.com/opentdf/java-sdk/releases/tag/{tag}" ) raise - except Exception as e: + except (urllib.error.URLError, OSError) as e: print(f" Warning: Could not verify artifact availability: {e}", file=sys.stderr) # Proceed with download attempt anyway diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py index 0712d05d..7a9d3295 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/java_fixup.py @@ -33,7 +33,7 @@ def post_checkout_java_fixup(base_dir: Path | None = None) -> None: # Extract version from directory name (e.g., "v0.7.5" -> "0.7.5") dir_name = src_dir.name - version = dir_name.lstrip("v") + version = dir_name.removeprefix("v") platform_branch = _get_platform_branch(version) pom_content = pom_file.read_text() diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 75a9a1fa..a7a29600 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -80,14 +80,14 @@ runs: shell: bash run: | SDK_MGR_DIR="$(cd "${{ inputs.path }}/../.." && pwd)/otdf-sdk-mgr" - for row in $(echo "${version_info}" | jq -c '.[]'); do + echo "${version_info}" | jq -c '.[]' | while IFS= read -r row; do tag=$(echo "$row" | jq -r '.tag') head=$(echo "$row" | jq -r '.head // false') release=$(echo "$row" | jq -r '.release // empty') if [[ "$head" != "true" && -n "$release" ]]; then echo "Installing ${{ inputs.sdk }} $tag from registry (release: $release)" - # Sanitize tag for use as an env var name (replace . with _) - tag_sanitized="${tag//./_}" + # Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _) + tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ --sdk "${{ inputs.sdk }}" --version "$release" \ --dist-name "$tag"; then From f1e54dbb9d44cad02e8f84996d1eee6c8ce40eec Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 12:11:28 -0500 Subject: [PATCH 13/24] fix(xtest): split space-separated ref aliases before spawnSync call spawnSync does not perform shell word-splitting, so passing "main latest" as a single string only resolved one tag instead of both. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/xtest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index ab5134b8..163cb0d8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -149,7 +149,8 @@ jobs: for (const [sdkType, ref] of Object.entries(refs)) { try { - const result = spawnSync('uv', ['run', '--project', sdkMgrDir, 'otdf-sdk-mgr', 'versions', 'resolve', sdkType, ref], { cwd: sdkMgrDir, encoding: 'utf-8' }); + const refArgs = ref.trim().split(/\s+/).filter(Boolean); + const result = spawnSync('uv', ['run', '--project', sdkMgrDir, 'otdf-sdk-mgr', 'versions', 'resolve', sdkType, ...refArgs], { cwd: sdkMgrDir, encoding: 'utf-8' }); if (result.status !== 0) { throw new Error(result.stderr || `Process exited with code ${result.status}`); } From f8f8642450cf1bfca144a244a3c3f6de362e4df6 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 12:14:28 -0500 Subject: [PATCH 14/24] chore(xtest): restore ci lint check --- .github/workflows/check.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index c147d24d..9706e5e4 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,6 +34,14 @@ jobs: uv run ruff format --check . uv run pyright working-directory: xtest + - name: Lint and test otdf-local + run: | + uv sync + uv run ruff check . + uv run ruff format --check . + uv run pyright + uv run pytest --maxfail=1 --disable-warnings -v --tb=short -m "not integration" + working-directory: otdf-local - name: Lint and test otdf-sdk-mgr run: | uv sync From d8a3b0bfe4234264033a94609adb3a78eb471372 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 12:18:06 -0500 Subject: [PATCH 15/24] chore(docs) update path --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 46ac5c7d..b48dd7dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ This guide provides essential knowledge for AI agents performing updates, refact ### Structure - **Test Directory**: `xtest/` - pytest-based integration tests -- **SDK Distributions**: `sdk/{go,java,js}/dist/` - built SDK distributions with CLI wrappers +- **SDK Distributions**: `xtest/sdk/{go,java,js}/dist/` - built SDK distributions with CLI wrappers - **SDK Configuration**: `otdf-sdk-mgr install` - installs SDK CLIs from released artifacts or delegates to source builds - **SDK Version Lookup**: `otdf-sdk-mgr versions list` - lists released artifacts across registries (Go git tags, npm, Maven Central, GitHub Releases) - **Platform**: `platform/` - OpenTDF platform service From 205aa3dd46e24db8d72e73e09ae4d8c2d2202ae7 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 13:04:43 -0500 Subject: [PATCH 16/24] Update checkout.py --- otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py index 618e4629..8d2eed71 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -20,8 +20,7 @@ def checkout_sdk_branch(language: str, branch: str) -> None: sdk_dirs = get_sdk_dirs() if language not in sdk_dirs: raise ValueError( - f"Unsupported language '{language}'. " - f"Supported values are: {', '.join(sdk_dirs)}" + f"Unsupported language '{language}'. Supported values are: {', '.join(sdk_dirs)}" ) sdk_dir = sdk_dirs[language] From f2d21275aca8235ab8b7388a2ef3a4d990b5f0fb Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 15:02:48 -0500 Subject: [PATCH 17/24] fix(resolve): resolve JS refs via npm; support pre-release and dist-tags - Add `_try_resolve_js_npm` helper that queries the npm registry for explicit JS version refs before falling back to git tag lookup. Pre-release tags (e.g. 0.9.0-beta.84) and npm dist-tags (e.g. next) now resolve to artifact installs without requiring a matching git tag. - Rename internal `listed_tags` to `stable_tags` after the semver filter so the unfiltered infix-stripped list remains available for fallback. - Fall back to the unfiltered tag list when a specific non-stable version is not found in stable tags (covers non-JS pre-release git tags). - Update workflow_dispatch input descriptions to show accepted ref formats and supported aliases (latest, lts) for each input. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/xtest.yml | 10 ++-- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 73 ++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 163cb0d8..fdc9a029 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -7,27 +7,27 @@ on: required: false type: string default: main - description: "The branch or commit to use for the platform" + description: "platform ref: branch, tag (e.g. service/v0.12.0), commit SHA, 'latest', or 'lts'" otdfctl-ref: required: false type: string default: main - description: "The branch or commit to use for otdfctl" + description: "otdfctl ref: branch, tag (e.g. v0.29.0), commit SHA, 'latest', or 'lts'" js-ref: required: false type: string default: main - description: "The branch or commit to use for the web-sdk" + description: "web-sdk ref: branch, tag (e.g. sdk/0.9.0 or 0.9.0-beta.84), commit SHA, 'latest', or 'lts'" java-ref: required: false type: string default: main - description: "The branch or commit to use for the java-sdk" + description: "java-sdk ref: branch, tag (e.g. v0.12.0), commit SHA, 'latest', or 'lts'" focus-sdk: required: false type: string default: all - description: "The SDK to focus on (go, js, java, all)" + description: "SDK to focus on (go, js, java, all)" workflow_call: inputs: platform-ref: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index d6d03006..ce4b0940 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -11,6 +11,7 @@ JAVA_PLATFORM_BRANCH_MAP, LTS_VERSIONS, SDK_GIT_URLS, + SDK_NPM_PACKAGES, ) @@ -51,6 +52,53 @@ def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: SHA_REGEX = r"^[a-f0-9]{7,40}$" +def _try_resolve_js_npm( + sdk: str, + version: str, + alias: str, + infix_stripped_tags: list[tuple[str, str]], + infix: str | None, +) -> ResolveSuccess | None: + """Try to resolve a JS version from the npm registry. + + Returns a ResolveSuccess if the version exists on npm, None otherwise. + The sha is populated from git tags on a best-effort basis (empty string if not found), + since it is only needed for source checkouts which are skipped for artifact installs. + """ + from otdf_sdk_mgr.registry import fetch_json + + package = SDK_NPM_PACKAGES.get(sdk) + if not package: + return None + + npm_version = version.lstrip("v") + try: + data = fetch_json(f"https://registry.npmjs.org/{package}/{npm_version}") + except Exception: + return None + + # npm may resolve a dist-tag (e.g. "next") to a concrete version + resolved_version = data.get("version", npm_version) + tag = resolved_version + + # Look up SHA from git tags (best-effort; not required for artifact installs) + sha = "" + candidates = {resolved_version, f"v{resolved_version}"} + for (s, t) in infix_stripped_tags: + if t in candidates: + sha = s + break + + release = f"{infix}/{tag}" if infix else tag + return { + "sdk": sdk, + "alias": alias, + "release": release, + "sha": sha, + "tag": tag, + } + + def lookup_additional_options(sdk: str, version: str) -> str | None: """Look up additional build options for a given SDK version.""" if sdk != "java": @@ -208,9 +256,17 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: for (sha, tag) in listed_tags if f"{infix}/" in tag ] + + # For JS: explicit version refs resolve via npm first so that pre-release and + # dist-tag refs (e.g. "0.9.0-beta.84", "next") work without a matching git tag. + if sdk in SDK_NPM_PACKAGES and version not in ("latest", "lts"): + npm_result = _try_resolve_js_npm(sdk, version, version, listed_tags, infix) + if npm_result is not None: + return npm_result + semver_regex = r"v?\d+\.\d+\.\d+$" - listed_tags = [(sha, tag) for (sha, tag) in listed_tags if re.search(semver_regex, tag)] - listed_tags.sort(key=lambda item: list(map(int, item[1].strip("v").split(".")))) + stable_tags = [(sha, tag) for (sha, tag) in listed_tags if re.search(semver_regex, tag)] + stable_tags.sort(key=lambda item: list(map(int, item[1].strip("v").split(".")))) alias = version matching_tags = [] if version == "latest": @@ -226,12 +282,12 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: latest_with_cli_tag = versions_with_cli[-1]["version"] matching_tags = [ (sha, tag) - for (sha, tag) in listed_tags + for (sha, tag) in stable_tags if tag in [latest_with_cli_tag, latest_with_cli_tag.lstrip("v")] ] if not matching_tags: # No versions with CLI found, fall back to building latest from source - sha, tag = listed_tags[-1] + sha, tag = stable_tags[-1] return { "sdk": sdk, "alias": alias, @@ -240,7 +296,7 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: "tag": tag, } else: - matching_tags = listed_tags[-1:] + matching_tags = stable_tags[-1:] else: if version == "lts": if sdk not in LTS_VERSIONS: @@ -250,8 +306,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: ) version = LTS_VERSIONS[sdk] matching_tags = [ - (sha, tag) for (sha, tag) in listed_tags if tag in [version, f"v{version}"] + (sha, tag) for (sha, tag) in stable_tags if tag in [version, f"v{version}"] ] + # If not found in stable tags, also search all tags (supports pre-release versions) + if not matching_tags: + matching_tags = [ + (sha, tag) for (sha, tag) in listed_tags if tag in [version, f"v{version}"] + ] if not matching_tags: raise ValueError(f"Tag [{version}] not found in [{sdk_url}]") sha, tag = matching_tags[-1] From 616b4e7f4f8fbff7ca869e12a839d53ebb0c320c Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 15:05:47 -0500 Subject: [PATCH 18/24] feat(ci): add Artifact column to Versions under Test summary table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Links to the upstream artifact registry for each SDK release: - JS → npmjs.com (/package/@opentdf/ctl/v/{version}) - Java → Maven Central (central.sonatype.com artifact page) - Go → pkg.go.dev (github.com/opentdf/otdfctl@{tag}) Source/head builds and platform entries show N/A (no registry artifact). Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/xtest.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index fdc9a029..77344889 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -170,10 +170,28 @@ jobs: core.summary.addHeading('Versions under Test', 3); + function artifactLink(sdkType, tag, release, head) { + if (head || !release) return ''; + const v = tag.replace(/^v/, ''); + if (sdkType === 'js') { + const url = `https://www.npmjs.com/package/@opentdf/ctl/v/${encodeURIComponent(v)}`; + return `npmjs`; + } + if (sdkType === 'java') { + const url = `https://central.sonatype.com/artifact/io.opentdf.platform/sdk/${encodeURIComponent(v)}`; + return `Maven Central`; + } + if (sdkType === 'go') { + const url = `https://pkg.go.dev/github.com/opentdf/otdfctl@${encodeURIComponent(tag)}`; + return `pkg.go.dev`; + } + return ''; + } + let errorCount = 0; const table = []; const th = (data) => ({ data, header: true }); - table.push([th('Library'), th('Tag'), th('SHA'), th('Alias'), th('Error')]); + table.push([th('Library'), th('Tag'), th('SHA'), th('Alias'), th('Artifact'), th('Error')]); for (const [sdkType, refInfo] of Object.entries(versionData)) { const tagList = []; @@ -185,9 +203,10 @@ jobs: const sdkLink = `${htmlEscape(sdkType)}`; const commitLink = sha ? `${htmlEscape(sha.substring(0, 7))}` : ' . '; const tagLink = (release && tag) - ? `${htmlEscape(tag)}` + ? `${htmlEscape(tag)}` : tag ? htmlEscape(tag) : 'N/A'; - table.push([sdkLink, tagLink, commitLink, alias || 'N/A', err || 'N/A']); + const artifactCell = artifactLink(sdkType, tag, release, head); + table.push([sdkLink, tagLink, commitLink, alias || 'N/A', artifactCell || 'N/A', err || 'N/A']); if (err) { errorCount += 1; continue; From 971c9883f553f5d20545aa4a237f53fbf9652f1a Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 18:12:15 -0500 Subject: [PATCH 19/24] Update resolve.py --- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index ce4b0940..7358038f 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -84,7 +84,7 @@ def _try_resolve_js_npm( # Look up SHA from git tags (best-effort; not required for artifact installs) sha = "" candidates = {resolved_version, f"v{resolved_version}"} - for (s, t) in infix_stripped_tags: + for s, t in infix_stripped_tags: if t in candidates: sha = s break From c0b665de185727224ca4b3be833445a702677174 Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Wed, 18 Feb 2026 18:20:16 -0500 Subject: [PATCH 20/24] fix(setup-cli-tool): fix tag sanitization for pre-release version env var names The check-source step used \`\${tag//./_}\` (dots only), leaving hyphens in place and producing invalid bash variable names like BUILD_FROM_SOURCE_0_9_0-beta_84. Match the install step's pattern \`\${tag//[^a-zA-Z0-9_]/_}\` to replace all non-alphanumeric/underscore characters. Co-Authored-By: Claude Sonnet 4.6 --- xtest/setup-cli-tool/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index a7a29600..9e110ef4 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -120,7 +120,7 @@ runs: fi tag=$(echo "$row" | jq -r '.tag') head=$(echo "$row" | jq -r '.head // false') - tag_sanitized="${tag//./_}" + tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" build_from_source_var="BUILD_FROM_SOURCE_${tag_sanitized}" if [[ "$head" == "true" || "${!build_from_source_var}" == "true" ]]; then echo "needs-source-${slot}=true" >> "$GITHUB_OUTPUT" From 5db2709bd07a9bf448548e9d4cc89c93e428132c Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 19 Feb 2026 08:55:44 -0500 Subject: [PATCH 21/24] test(otdf-sdk-mgr): add pytest coverage for semver and resolve modules Add 50 tests covering semver.py (pure functions) and resolve.py (mocked git/network) to catch edge cases in SHA disambiguation, merge queue parsing, JS npm fallback, and Java latest-with-CLI logic. Fix bugs found during test authoring: - semver.py: support build metadata in SEMVER_RE (1.2.3-pre+build) - resolve.py: extend SHA_REGEX to 64 chars to accept SHA-256 - resolve.py: move sdk_url lookup inside try/except so unknown SDKs return ResolveError instead of raising KeyError Co-Authored-By: Claude Sonnet 4.6 --- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 4 +- otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py | 2 +- otdf-sdk-mgr/tests/__init__.py | 0 otdf-sdk-mgr/tests/test_resolve.py | 305 +++++++++++++++++++++++ otdf-sdk-mgr/tests/test_semver.py | 93 +++++++ 5 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 otdf-sdk-mgr/tests/__init__.py create mode 100644 otdf-sdk-mgr/tests/test_resolve.py create mode 100644 otdf-sdk-mgr/tests/test_semver.py diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 7358038f..6e4cd7ca 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -49,7 +49,7 @@ def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: r"^refs/heads/gh-readonly-queue/(?P[^/]+)/pr-(?P\d+)-(?P[a-f0-9]{40})$" ) -SHA_REGEX = r"^[a-f0-9]{7,40}$" +SHA_REGEX = r"^[a-f0-9]{7,64}$" def _try_resolve_js_npm( @@ -113,8 +113,8 @@ def lookup_additional_options(sdk: str, version: str) -> str | None: def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: """Resolve a version spec to a concrete SHA and tag.""" - sdk_url = SDK_GIT_URLS[sdk] try: + sdk_url = SDK_GIT_URLS[sdk] repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py index 74581a7c..6103f362 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/semver.py @@ -4,7 +4,7 @@ import re -SEMVER_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-(.+))?$") +SEMVER_RE = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)(?:-([^+]+))?(?:\+(.+))?$") def parse_semver(version: str) -> tuple[int, int, int, str | None] | None: diff --git a/otdf-sdk-mgr/tests/__init__.py b/otdf-sdk-mgr/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py new file mode 100644 index 00000000..bff27076 --- /dev/null +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -0,0 +1,305 @@ +"""Tests for resolve.py — mocks git.Git to avoid network calls.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from otdf_sdk_mgr.resolve import ( + _try_resolve_js_npm, + is_resolve_error, + is_resolve_success, + resolve, +) + +SHA40 = "a" * 40 +SHA64 = "b" * 64 +SHA7 = "c" * 7 + +# A realistic ls_remote output (tab-separated "sha\tref" per line) +def make_ls_remote(*entries): + """Build a ls_remote string from (sha, ref) pairs.""" + return "\n".join(f"{sha}\t{ref}" for sha, ref in entries) + + +def patch_git(ls_remote_output): + mock_git = MagicMock() + mock_git.ls_remote.return_value = ls_remote_output + return patch("otdf_sdk_mgr.resolve.Git", return_value=mock_git) + + +# --------------------------------------------------------------------------- +# Type guards +# --------------------------------------------------------------------------- + +class TestTypeGuards: + def test_is_resolve_error(self): + err = {"sdk": "go", "alias": "x", "err": "oops"} + assert is_resolve_error(err) is True + assert is_resolve_success(err) is False + + def test_is_resolve_success(self): + ok = {"sdk": "go", "alias": "x", "sha": SHA40, "tag": "v1.0.0"} + assert is_resolve_success(ok) is True + assert is_resolve_error(ok) is False + + +# --------------------------------------------------------------------------- +# resolve() — "main" +# --------------------------------------------------------------------------- + +class TestResolveMain: + def test_main_returns_head(self): + ls = make_ls_remote((SHA40, "refs/heads/main")) + with patch_git(ls): + result = resolve("go", "main", None) + assert is_resolve_success(result) + assert result["head"] is True + assert result["tag"] == "main" + assert result["sha"] == SHA40 + + def test_refs_heads_main_alias(self): + ls = make_ls_remote((SHA40, "refs/heads/main")) + with patch_git(ls): + result = resolve("go", "refs/heads/main", None) + assert is_resolve_success(result) + assert result["tag"] == "main" + + +# --------------------------------------------------------------------------- +# resolve() — SHA inputs +# --------------------------------------------------------------------------- + +class TestResolveSHA: + def test_sha_no_matches_returns_sha_as_tag(self): + ls = make_ls_remote(("d" * 40, "refs/heads/other")) + with patch_git(ls): + result = resolve("go", SHA7, None) + assert is_resolve_success(result) + assert result["sha"] == SHA7 + assert result["tag"] == SHA7 + + def test_sha1_short_matches(self): + ls = make_ls_remote((SHA40, "refs/tags/v1.0.0")) + with patch_git(ls): + result = resolve("go", SHA7, None) + # SHA7 = "ccc..." but SHA40 = "aaa..." so no match; override for this test + ls2 = make_ls_remote((SHA7 + "0" * 33, "refs/tags/v1.0.0")) + with patch_git(ls2): + result = resolve("go", SHA7, None) + assert is_resolve_success(result) + assert result["tag"] == "v1.0.0" + + def test_sha1_full_matches(self): + ls = make_ls_remote((SHA40, "refs/tags/v2.0.0")) + with patch_git(ls): + result = resolve("go", SHA40, None) + assert is_resolve_success(result) + assert result["tag"] == "v2.0.0" + + def test_sha256_full_matches(self): + ls = make_ls_remote((SHA64, "refs/tags/v3.0.0")) + with patch_git(ls): + result = resolve("go", SHA64, None) + assert is_resolve_success(result) + assert result["tag"] == "v3.0.0" + + def test_single_match_strips_refs_tags(self): + ls = make_ls_remote((SHA40, "refs/tags/v1.2.3")) + with patch_git(ls): + result = resolve("go", SHA40, None) + assert result["tag"] == "v1.2.3" + + def test_multiple_matches_pr_takes_priority(self): + other_sha = "e" * 40 + ls = make_ls_remote( + (SHA40, "refs/pull/99/head"), + (SHA40, "refs/heads/some-branch"), + ) + with patch_git(ls): + result = resolve("go", SHA40, None) + assert is_resolve_success(result) + assert result["tag"] == "pull-99" + + def test_multiple_matches_merge_queue(self): + mq_ref = f"refs/heads/gh-readonly-queue/main/pr-42-{SHA40}" + ls = make_ls_remote( + (SHA40, mq_ref), + (SHA40, "refs/heads/main"), + ) + with patch_git(ls): + result = resolve("go", SHA40, None) + assert is_resolve_success(result) + assert result["tag"] == "mq-main-42" + assert result["pr"] == "42" + + def test_multiple_matches_branch_only(self): + ls = make_ls_remote( + (SHA40, "refs/heads/feature/my-branch"), + (SHA40, "refs/heads/main"), + ) + with patch_git(ls): + result = resolve("go", SHA40, None) + assert is_resolve_success(result) + assert result["head"] is True + assert result["tag"] == "feature--my-branch" + + +# --------------------------------------------------------------------------- +# resolve() — refs/pull/NNN +# --------------------------------------------------------------------------- + +class TestResolvePR: + def test_pr_found(self): + # The code filters rows where r.endswith(version), so the ref must end with "refs/pull/123" + ls = make_ls_remote((SHA40, "refs/pull/123")) + with patch_git(ls): + result = resolve("go", "refs/pull/123", None) + assert is_resolve_success(result) + assert result["pr"] == "123" + assert result["tag"] == "pull-123" + assert result["head"] is True + + def test_pr_not_found(self): + ls = make_ls_remote((SHA40, "refs/heads/main")) + with patch_git(ls): + result = resolve("go", "refs/pull/999", None) + assert is_resolve_error(result) + + +# --------------------------------------------------------------------------- +# resolve() — branch name +# --------------------------------------------------------------------------- + +class TestResolveBranch: + def test_exact_branch_match(self): + ls = make_ls_remote( + (SHA40, "refs/heads/my-feature"), + (SHA40, "refs/heads/main"), + ) + with patch_git(ls): + result = resolve("go", "my-feature", None) + assert is_resolve_success(result) + assert result["head"] is True + assert result["tag"] == "my-feature" + + +# --------------------------------------------------------------------------- +# resolve() — version tags +# --------------------------------------------------------------------------- + +class TestResolveVersionTags: + def test_exact_stable_version(self): + ls = make_ls_remote( + (SHA40, "refs/tags/v0.3.5"), + ("0" * 40, "refs/tags/v0.3.4"), + ) + with patch_git(ls): + result = resolve("go", "v0.3.5", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.3.5" + assert result["sha"] == SHA40 + + def test_pre_release_version_fallback(self): + ls = make_ls_remote( + (SHA40, "refs/tags/v0.3.5-beta.1"), + ) + with patch_git(ls): + result = resolve("go", "v0.3.5-beta.1", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.3.5-beta.1" + + def test_lts_resolves_to_config_version(self): + from otdf_sdk_mgr.config import LTS_VERSIONS + lts_ver = LTS_VERSIONS["go"] + ls = make_ls_remote( + (SHA40, f"refs/tags/v{lts_ver}"), + ) + with patch_git(ls): + result = resolve("go", "lts", None) + assert is_resolve_success(result) + assert result["tag"] in [lts_ver, f"v{lts_ver}"] + + def test_lts_unknown_sdk_raises(self): + # SDK not in SDK_GIT_URLS → KeyError, caught → ResolveError + result = resolve("unknownsdk", "lts", None) + assert is_resolve_error(result) + + +# --------------------------------------------------------------------------- +# resolve() — "latest" +# --------------------------------------------------------------------------- + +class TestResolveLatest: + def test_non_java_returns_last_stable(self): + ls = make_ls_remote( + ("1" * 40, "refs/tags/v0.1.0"), + ("2" * 40, "refs/tags/v0.2.0"), + ("3" * 40, "refs/tags/v0.3.0"), + ) + with patch_git(ls): + result = resolve("go", "latest", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.3.0" + + def test_java_with_cli_available(self): + ls = make_ls_remote( + ("1" * 40, "refs/tags/v0.1.0"), + ("2" * 40, "refs/tags/v0.2.0"), + ) + mock_releases = [ + {"version": "v0.1.0", "has_cli": False}, + {"version": "v0.2.0", "has_cli": True}, + ] + with patch_git(ls), patch( + "otdf_sdk_mgr.registry.list_java_github_releases", return_value=mock_releases + ): + result = resolve("java", "latest", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.2.0" + + def test_java_no_cli_available_falls_back_to_source(self): + ls = make_ls_remote( + ("1" * 40, "refs/tags/v0.1.0"), + ("2" * 40, "refs/tags/v0.2.0"), + ) + mock_releases = [ + {"version": "v0.1.0", "has_cli": False}, + {"version": "v0.2.0", "has_cli": False}, + ] + with patch_git(ls), patch( + "otdf_sdk_mgr.registry.list_java_github_releases", return_value=mock_releases + ): + result = resolve("java", "latest", None) + assert is_resolve_success(result) + assert result.get("head") is True + + +# --------------------------------------------------------------------------- +# _try_resolve_js_npm() +# --------------------------------------------------------------------------- + +class TestTryResolveJsNpm: + def test_npm_concrete_version(self): + tags = [(SHA40, "v1.2.3"), ("0" * 40, "v1.2.2")] + with patch("otdf_sdk_mgr.registry.fetch_json", return_value={"version": "1.2.3"}): + result = _try_resolve_js_npm("js", "1.2.3", "1.2.3", tags, None) + assert result is not None + assert result["tag"] == "1.2.3" + assert result["sha"] == SHA40 + + def test_npm_dist_tag_resolved(self): + tags = [(SHA40, "v0.9.0")] + with patch("otdf_sdk_mgr.registry.fetch_json", return_value={"version": "0.9.0"}): + result = _try_resolve_js_npm("js", "next", "next", tags, None) + assert result is not None + assert result["tag"] == "0.9.0" + assert result["sha"] == SHA40 + + def test_npm_raises_returns_none(self): + with patch("otdf_sdk_mgr.registry.fetch_json", side_effect=Exception("network error")): + result = _try_resolve_js_npm("js", "1.2.3", "1.2.3", [], None) + assert result is None + + def test_unknown_sdk_returns_none(self): + result = _try_resolve_js_npm("go", "1.2.3", "1.2.3", [], None) + assert result is None diff --git a/otdf-sdk-mgr/tests/test_semver.py b/otdf-sdk-mgr/tests/test_semver.py new file mode 100644 index 00000000..c6178e76 --- /dev/null +++ b/otdf-sdk-mgr/tests/test_semver.py @@ -0,0 +1,93 @@ +"""Tests for semver.py — pure functions, no mocking needed.""" + +import pytest + +from otdf_sdk_mgr.semver import is_stable, normalize_version, parse_semver, semver_sort_key + + +class TestParseSemver: + def test_basic(self): + assert parse_semver("1.2.3") == (1, 2, 3, None) + + def test_v_prefix(self): + assert parse_semver("v1.2.3") == (1, 2, 3, None) + + def test_pre_release(self): + assert parse_semver("1.2.3-beta.1") == (1, 2, 3, "beta.1") + + def test_zeros(self): + assert parse_semver("0.0.0") == (0, 0, 0, None) + + def test_build_metadata_only(self): + # build metadata is parsed but discarded; pre stays None + assert parse_semver("1.2.3+build.1") == (1, 2, 3, None) + + def test_pre_and_build(self): + assert parse_semver("1.2.3-pre+build") == (1, 2, 3, "pre") + + def test_invalid_branch(self): + assert parse_semver("main") is None + + def test_invalid_empty(self): + assert parse_semver("") is None + + def test_invalid_partial(self): + assert parse_semver("v1.2") is None + + def test_invalid_alpha(self): + assert parse_semver("abc") is None + + +class TestIsStable: + def test_stable(self): + assert is_stable("1.2.3") is True + + def test_v_stable(self): + assert is_stable("v1.2.3") is True + + def test_pre_release(self): + assert is_stable("1.2.3-beta.1") is False + + def test_rc(self): + assert is_stable("1.2.3-rc.1") is False + + def test_build_metadata_only_is_stable(self): + # build metadata does not make a version pre-release + assert is_stable("1.2.3+build.42") is True + + def test_non_semver(self): + assert is_stable("main") is False + + def test_empty(self): + assert is_stable("") is False + + +class TestSemverSortKey: + def test_pre_before_stable_same_version(self): + pre = semver_sort_key("1.2.3-alpha") + stable = semver_sort_key("1.2.3") + assert pre < stable + + def test_cross_version_ordering(self): + assert semver_sort_key("1.0.0") < semver_sort_key("2.0.0") + assert semver_sort_key("1.2.3") < semver_sort_key("1.2.4") + assert semver_sort_key("1.9.0") < semver_sort_key("1.10.0") + + def test_non_semver_fallback(self): + key = semver_sort_key("main") + assert key == (0, 0, 0, 0, "main") + + def test_build_metadata_ignored_for_ordering(self): + # 1.2.3+build should sort same as 1.2.3 (both stable, same version) + assert semver_sort_key("1.2.3+build") == semver_sort_key("1.2.3") + + +class TestNormalizeVersion: + def test_adds_v_prefix(self): + assert normalize_version("1.2.3") == "v1.2.3" + + def test_preserves_v_prefix(self): + assert normalize_version("v1.2.3") == "v1.2.3" + + def test_strips_whitespace(self): + assert normalize_version(" 1.2.3 ") == "v1.2.3" From 085b32921d71b7bb1501dd5f29c88af0dfa362aa Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 19 Feb 2026 08:58:54 -0500 Subject: [PATCH 22/24] fix(test): remove unused variable in test_resolve.py Co-Authored-By: Claude Sonnet 4.6 --- otdf-sdk-mgr/tests/test_resolve.py | 2 -- otdf-sdk-mgr/tests/test_semver.py | 1 - 2 files changed, 3 deletions(-) diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index bff27076..3074cbf9 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch -import pytest from otdf_sdk_mgr.resolve import ( _try_resolve_js_npm, @@ -110,7 +109,6 @@ def test_single_match_strips_refs_tags(self): assert result["tag"] == "v1.2.3" def test_multiple_matches_pr_takes_priority(self): - other_sha = "e" * 40 ls = make_ls_remote( (SHA40, "refs/pull/99/head"), (SHA40, "refs/heads/some-branch"), diff --git a/otdf-sdk-mgr/tests/test_semver.py b/otdf-sdk-mgr/tests/test_semver.py index c6178e76..fbee25fb 100644 --- a/otdf-sdk-mgr/tests/test_semver.py +++ b/otdf-sdk-mgr/tests/test_semver.py @@ -1,6 +1,5 @@ """Tests for semver.py — pure functions, no mocking needed.""" -import pytest from otdf_sdk_mgr.semver import is_stable, normalize_version, parse_semver, semver_sort_key From 20562ea8a753914bf4bed2b83e6a22037dde056e Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 19 Feb 2026 09:11:03 -0500 Subject: [PATCH 23/24] chore(xtest) ruff format --- otdf-sdk-mgr/tests/test_resolve.py | 20 ++++++++++++++++---- otdf-sdk-mgr/tests/test_semver.py | 1 - 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index 3074cbf9..3dca8071 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -14,6 +14,7 @@ SHA64 = "b" * 64 SHA7 = "c" * 7 + # A realistic ls_remote output (tab-separated "sha\tref" per line) def make_ls_remote(*entries): """Build a ls_remote string from (sha, ref) pairs.""" @@ -30,6 +31,7 @@ def patch_git(ls_remote_output): # Type guards # --------------------------------------------------------------------------- + class TestTypeGuards: def test_is_resolve_error(self): err = {"sdk": "go", "alias": "x", "err": "oops"} @@ -46,6 +48,7 @@ def test_is_resolve_success(self): # resolve() — "main" # --------------------------------------------------------------------------- + class TestResolveMain: def test_main_returns_head(self): ls = make_ls_remote((SHA40, "refs/heads/main")) @@ -68,6 +71,7 @@ def test_refs_heads_main_alias(self): # resolve() — SHA inputs # --------------------------------------------------------------------------- + class TestResolveSHA: def test_sha_no_matches_returns_sha_as_tag(self): ls = make_ls_remote(("d" * 40, "refs/heads/other")) @@ -146,6 +150,7 @@ def test_multiple_matches_branch_only(self): # resolve() — refs/pull/NNN # --------------------------------------------------------------------------- + class TestResolvePR: def test_pr_found(self): # The code filters rows where r.endswith(version), so the ref must end with "refs/pull/123" @@ -168,6 +173,7 @@ def test_pr_not_found(self): # resolve() — branch name # --------------------------------------------------------------------------- + class TestResolveBranch: def test_exact_branch_match(self): ls = make_ls_remote( @@ -185,6 +191,7 @@ def test_exact_branch_match(self): # resolve() — version tags # --------------------------------------------------------------------------- + class TestResolveVersionTags: def test_exact_stable_version(self): ls = make_ls_remote( @@ -208,6 +215,7 @@ def test_pre_release_version_fallback(self): def test_lts_resolves_to_config_version(self): from otdf_sdk_mgr.config import LTS_VERSIONS + lts_ver = LTS_VERSIONS["go"] ls = make_ls_remote( (SHA40, f"refs/tags/v{lts_ver}"), @@ -227,6 +235,7 @@ def test_lts_unknown_sdk_raises(self): # resolve() — "latest" # --------------------------------------------------------------------------- + class TestResolveLatest: def test_non_java_returns_last_stable(self): ls = make_ls_remote( @@ -248,8 +257,9 @@ def test_java_with_cli_available(self): {"version": "v0.1.0", "has_cli": False}, {"version": "v0.2.0", "has_cli": True}, ] - with patch_git(ls), patch( - "otdf_sdk_mgr.registry.list_java_github_releases", return_value=mock_releases + with ( + patch_git(ls), + patch("otdf_sdk_mgr.registry.list_java_github_releases", return_value=mock_releases), ): result = resolve("java", "latest", None) assert is_resolve_success(result) @@ -264,8 +274,9 @@ def test_java_no_cli_available_falls_back_to_source(self): {"version": "v0.1.0", "has_cli": False}, {"version": "v0.2.0", "has_cli": False}, ] - with patch_git(ls), patch( - "otdf_sdk_mgr.registry.list_java_github_releases", return_value=mock_releases + with ( + patch_git(ls), + patch("otdf_sdk_mgr.registry.list_java_github_releases", return_value=mock_releases), ): result = resolve("java", "latest", None) assert is_resolve_success(result) @@ -276,6 +287,7 @@ def test_java_no_cli_available_falls_back_to_source(self): # _try_resolve_js_npm() # --------------------------------------------------------------------------- + class TestTryResolveJsNpm: def test_npm_concrete_version(self): tags = [(SHA40, "v1.2.3"), ("0" * 40, "v1.2.2")] diff --git a/otdf-sdk-mgr/tests/test_semver.py b/otdf-sdk-mgr/tests/test_semver.py index fbee25fb..ce89eab7 100644 --- a/otdf-sdk-mgr/tests/test_semver.py +++ b/otdf-sdk-mgr/tests/test_semver.py @@ -1,6 +1,5 @@ """Tests for semver.py — pure functions, no mocking needed.""" - from otdf_sdk_mgr.semver import is_stable, normalize_version, parse_semver, semver_sort_key From 3426af71674e6f3deb11a2d186ea66cab00c6a2b Mon Sep 17 00:00:00 2001 From: David Mihalcik Date: Thu, 19 Feb 2026 09:34:23 -0500 Subject: [PATCH 24/24] fix(test): resolve pyright type errors in test_resolve.py - Cast plain dicts to ResolveResult in type guard tests - Import ResolveResult for cast - Add missing is_resolve_success() guard before result["tag"] access - Use result.get() for NotRequired keys (head, pr) Co-Authored-By: Claude Sonnet 4.6 --- otdf-sdk-mgr/tests/test_resolve.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index 3dca8071..f100152f 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -1,9 +1,10 @@ """Tests for resolve.py — mocks git.Git to avoid network calls.""" +from typing import cast from unittest.mock import MagicMock, patch - from otdf_sdk_mgr.resolve import ( + ResolveResult, _try_resolve_js_npm, is_resolve_error, is_resolve_success, @@ -34,12 +35,12 @@ def patch_git(ls_remote_output): class TestTypeGuards: def test_is_resolve_error(self): - err = {"sdk": "go", "alias": "x", "err": "oops"} + err = cast(ResolveResult, {"sdk": "go", "alias": "x", "err": "oops"}) assert is_resolve_error(err) is True assert is_resolve_success(err) is False def test_is_resolve_success(self): - ok = {"sdk": "go", "alias": "x", "sha": SHA40, "tag": "v1.0.0"} + ok = cast(ResolveResult, {"sdk": "go", "alias": "x", "sha": SHA40, "tag": "v1.0.0"}) assert is_resolve_success(ok) is True assert is_resolve_error(ok) is False @@ -55,7 +56,7 @@ def test_main_returns_head(self): with patch_git(ls): result = resolve("go", "main", None) assert is_resolve_success(result) - assert result["head"] is True + assert result.get("head") is True assert result["tag"] == "main" assert result["sha"] == SHA40 @@ -110,6 +111,7 @@ def test_single_match_strips_refs_tags(self): ls = make_ls_remote((SHA40, "refs/tags/v1.2.3")) with patch_git(ls): result = resolve("go", SHA40, None) + assert is_resolve_success(result) assert result["tag"] == "v1.2.3" def test_multiple_matches_pr_takes_priority(self): @@ -132,7 +134,7 @@ def test_multiple_matches_merge_queue(self): result = resolve("go", SHA40, None) assert is_resolve_success(result) assert result["tag"] == "mq-main-42" - assert result["pr"] == "42" + assert result.get("pr") == "42" def test_multiple_matches_branch_only(self): ls = make_ls_remote( @@ -142,7 +144,7 @@ def test_multiple_matches_branch_only(self): with patch_git(ls): result = resolve("go", SHA40, None) assert is_resolve_success(result) - assert result["head"] is True + assert result.get("head") is True assert result["tag"] == "feature--my-branch" @@ -158,9 +160,9 @@ def test_pr_found(self): with patch_git(ls): result = resolve("go", "refs/pull/123", None) assert is_resolve_success(result) - assert result["pr"] == "123" + assert result.get("pr") == "123" assert result["tag"] == "pull-123" - assert result["head"] is True + assert result.get("head") is True def test_pr_not_found(self): ls = make_ls_remote((SHA40, "refs/heads/main")) @@ -183,7 +185,7 @@ def test_exact_branch_match(self): with patch_git(ls): result = resolve("go", "my-feature", None) assert is_resolve_success(result) - assert result["head"] is True + assert result.get("head") is True assert result["tag"] == "my-feature"