From fbb1a17071cb6aef42717a84c904a5a92e7bc693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Wr=C3=B3bel?= Date: Fri, 3 Jul 2026 12:00:50 +0200 Subject: [PATCH 1/2] BitBucket support - plan & discussion --- .../workspace-repos/bitbucket-support.md | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/plans/workspace-repos/bitbucket-support.md diff --git a/docs/plans/workspace-repos/bitbucket-support.md b/docs/plans/workspace-repos/bitbucket-support.md new file mode 100644 index 0000000..0c288b3 --- /dev/null +++ b/docs/plans/workspace-repos/bitbucket-support.md @@ -0,0 +1,92 @@ +# Add BitBucket Support to SpecFlow + +## Context + +SpecFlow is hardwired to **GitHub** end-to-end with **no git-hosting-provider abstraction**. GitHub-specific logic is scattered across credential resolution, authenticated-URL construction, token sanitization, repo provisioning, deploy prompts (GitHub Actions + `gh` CLI), onboarding, and notifications. The only pre-existing notion of provider variance is a dormant `GitType` enum (`GITHUB`/`GITLAB`) in the P10Y models. `bitbucket` appears nowhere except the branch name — this is greenfield. + +We need to add BitBucket as a first-class git host. **Scope decisions (confirmed with user):** +1. **Full parity**: git hosting AND CI/CD (deploy/E2E). +2. **Both BitBucket Cloud (bitbucket.org) and Server/Data Center (self-hosted).** +3. **Auth = Access Tokens** (Repository/Workspace access tokens), not app passwords. +4. **Per-generation/per-key coexistence**: GitHub and BitBucket usable side-by-side; provider resolved per workspace / per API-key, mirroring the existing per-key encrypted-token model. + +**Outcome**: a `GitProvider` abstraction (Open/Closed + SRP) that generalizes the three currently-hardcoded single-source mechanisms — authenticated-URL builder, token sanitizer, deploy recipe — *in place*, plus new BitBucket implementations, so a workspace on any provider can clone/commit/push/archive, be provisioned, and (Cloud) deploy via Pipelines. + +## ⚠️ Design decision to confirm during execution: Server/DC CI/CD + +BitBucket **Server/Data Center has no native CI** (no Pipelines). "Full CI/CD parity" is achievable only for **Cloud** (Pipelines via REST API). For **Server/DC**, the recommended design **degrades gracefully**: git-hosting + codegen fully work, but the deploy/E2E loop is **skipped with a structured USER NOTICE** (reusing the existing DEPLOY FAILURE REPORT / Slack channel) telling the operator that Server/DC deploy needs external CI (Jenkins/Bamboo) SpecFlow doesn't own. A future `GenericWebhookDeployStrategy` (hitting a configured `DEPLOY_WEBHOOK_URL`) can slot behind the same interface without reopening the work. This plan proceeds on that basis. + +## The Abstraction + +New package `backend/app/services/git_providers/`, SRP-split across the three layers it touches (git subprocess, provisioning script, deploy/prompt). One `Enum` selects implementations via a registry. + +- `enums.py` — `GitProvider(str, Enum)`: `GITHUB`, `BITBUCKET_CLOUD`, `BITBUCKET_SERVER`. +- `hosting.py` — `GitHostingProvider(ABC)`: `authenticated_clone_url(repo_url, auth)`, `token_sanitization_patterns()`, `branch_web_url(repo_url, branch)`, `default_git_user()`. +- `provisioning.py` — `RepoProvisioner(ABC)`: `repository_exists`, `create_repository`, `grant_write_access`, `get_authenticated_actor`, `web_url_for`. +- `deploy_strategy.py` — `DeployStrategy(ABC)`: `supports_managed_ci` (ClassVar bool), `deploy_env(token)`, `build_deploy_recipe(ctx)`, `unsupported_ci_notice(ctx)`. +- `registry.py` — `hosting_for`, `deploy_strategy_for`, `provisioner_for`, `provider_from_repo_url(repo_url)` (host-based default: `bitbucket.org`→CLOUD, `github.com`→GITHUB, else SERVER-requires-explicit-config). +- `github.py`, `bitbucket.py` — concrete impls. + +Provider-neutral pieces to **REUSE unchanged**: `git_utils.run_git`, `git_archive_service.GitArchiveService` (operate on `origin`), per-workspace `repo_url` storage. + +**Persistence / back-compat** (must read existing data with zero migration required): +- Workspace doc gains `git_provider` (nullable; read fallback = `provider_from_repo_url(repo_url)` → `github` for existing URLs). Server/DC workspaces MUST carry it explicitly (host isn't inferable). +- API-key doc: read ciphertext new→legacy (`git_token_ciphertext` → `github_token_ciphertext`); write only new. Add `git_provider` on key doc (default `github` for legacy). Keep `PUT /api/v1/auth/github-token` route + add alias `PUT /api/v1/auth/git-token`; thin re-export shim for renamed modules for one release. + +## Phased Implementation + +Order deliberately lands **git-hosting before CI/CD** (CI/CD is riskiest). + +### Phase 0 — Scaffolding + GitHub-first (no behavior change) +Create the `git_providers/` package + interfaces + `GitHubProvider` that *moves* today's exact logic (the `user:token@` URL builder and `ghp_`/`ghs_`/`github_pat_`+`@github.com` regexes out of `workspace_pool.py`; the `{"GH_TOKEN": token}` env and `gh workflow run` recipe out of `agents_claude_code.py`). +- **Golden/characterization tests** (`backend/tests/unit/git_providers/test_github_provider.py`) assert byte-identical output vs current behavior — makes Phases 2 & 5 provably non-regressive. + +### Phase 1 — Persistence + auth/secret generalization +- Rename `backend/app/core/github_platform_secrets.py` → `git_platform_secrets.py` (keep re-export shim). `GithubPlatformSecrets` → `GitPlatformSecrets` holding Fernet + `dict[GitProvider, DefaultIdentity]`. New settings `GIT_TOKEN_DEFAULT_BITBUCKET` + K8s key `K8S_SECRET_KEY_BITBUCKET_DEFAULT`; keep `GITHUB_TOKEN_DEFAULT`/`GITHUB_TOKEN` aliases. +- `backend/app/services/github_auth.py` → generalize to `git_auth.py`: `GithubAuthContext` → `GitAuthContext(git_user_name, token, provider)`. `resolve_git_auth_for_api_key_document(doc, workspace_provider, secrets)` preserves order (per-key ciphertext → per-provider default → error). `github_cli_env_for_generation` → `deploy_env_for_generation` returning `strategy.deploy_env(token)`. Keep old names as deprecated aliases to stage callsite churn. +- Modify `backend/app/core/config.py` (new settings/K8s keys + aliases), `backend/app/core/app_lifecycle.py` (renamed init), `backend/app/api/v1/auth.py` (`GitTokenUploadBody`, write `git_token_ciphertext`+`git_provider`, accept old shape). +- Idempotent, read-safe migration `backend/scripts/migrate_git_provider.py` (backfill workspace `git_provider`; copy ciphertext field). Not required for reads. +- Tests: auth-resolution matrix (legacy-only / new-only / per-provider default / error), secrets loader from env + mocked K8s map, both providers. + +### Phase 2 — Clone / push / sanitize generalization (git-hosting parity lands) +- `backend/app/services/workspace_pool.py`: `_get_authenticated_repo_url` delegates to `hosting_for(auth.provider).authenticated_clone_url(...)` (delete inline string surgery). `_sanitize_token_in_message` keeps explicit-token replacement, then applies patterns from **all** registered providers (defensive — never leak even on misresolution). Every `resolve_github_auth_for_generation_id(...)` callsite → `resolve_git_auth_for_generation_id(...)` (loads workspace `git_provider`). Clone/init/archive/reset funcs otherwise unchanged (operate on `origin`/`run_git`). GitHub-specific auth-failure hint string becomes provider-parameterized. +- Create `backend/app/services/git_providers/bitbucket.py`: `BitbucketCloudProvider` (clone `https://x-token-auth:{token}@bitbucket.org/{ws}/{repo}.git`; branch `/src/{branch}`; sanitize `@bitbucket.org`, `x-token-auth:`, `ATCTT` prefix) and `BitbucketServerProvider` (clone `https://x-token-auth:{token}@{host}/scm/{proj}/{repo}.git`; branch `/browse?at=refs%2Fheads%2F{branch}`; host from `BITBUCKET_SERVER_BASE_URL`). +- Tests: URL injection + idempotence, BB sanitization shapes, branch-url builders, clone flow with BB workspace doc (mock `run_git`). + +### Phase 3 — Repo provisioning +- `backend/scripts/create_generation_session_repos.py`: extract `GitHubAPIClient` to `RepoProvisioner` (in `github.py`); add `BitbucketCloudProvisioner` (`https://api.bitbucket.org/2.0`, `POST /repositories/{ws}/{repo}`, permission endpoints, `Bearer` access token) and `BitbucketServerProvisioner` (`{base}/rest/api/1.0/projects/{key}/repos`). Add `--git-provider` flag; generalize `--github-org`/`--team` → `--namespace`/`--principal` (keep aliases). Emit `git_provider` + `provisioner.web_url_for(name)` as `repo_url` (no hardcoded `github.com`). `_normalize_git_url` unchanged. +- Tests: provisioner unit tests (mock `httpx`) per provider; workspace-doc emission. + +### Phase 4 — P10Y + notifications (small, low-risk) +- `backend/app/services/p10y/p10y_api_models.py`: add `GitType.BITBUCKET = "bitbucket"`; set `RepositoryDetails.git_provider` for BB repos. +- `backend/app/core/notifications.py` (~line 1144): replace hardcoded `{repo_url}/tree/{branch}` with `hosting_for(provider).branch_web_url(...)` (resolve provider from workspace doc). Third single-source site — generalize, don't branch beside it. + +### Phase 5 — CI/CD layer (riskiest — last) +Bifurcation: **GitHub** = existing `gh` recipe (now inside `GitHubProvider`); **BB Cloud** = `curl`+`jq` recipe (`POST /2.0/repositories/{ws}/{repo}/pipelines/` custom selector, poll `GET .../pipelines/{uuid}` for `state.name==COMPLETED`, config `bitbucket-pipelines.yml`) — curl+jq already in `bash_usage`, no new tooling; **BB Server/DC** = `supports_managed_ci=False`, deploy skipped with USER NOTICE (see top-of-plan design decision). +- `backend/app/schemas/deploy_context.py`: `DeployGithubContext` → `DeployContext(repo, ref, deploy_workflow, provider)` (keep alias). +- `backend/app/services/workflow_steps.py` `_build_deploy_github_context` (~1064): read workspace `git_provider`; if `not supports_managed_ci`, emit notice + skip (generalize the `github_repo` guard at ~1176 to capability+repo). +- `backend/app/prompts/agents_claude_code.py` `generate_deploy_phase_agent_template` (~1139): replace the hardcoded GitHub-Actions bash block with `strategy.build_deploy_recipe(ctx)` (GitHub block moves verbatim into `GitHubDeployStrategy`; BB Cloud block new). Planning-side text (~741) becomes provider-agnostic. +- `backend/app/services/claude_code.py` (~1178): `deploy_extra_env = deploy_env_for_generation(...)`. +- `backend/app/core/tool_usage.py`: add `deploy_extra_tools_for(provider)` gating `Bash(gh:*)` to GitHub deploys (BB uses curl/jq). +- `backend/app/services/agent_hooks.py` `_ci_blocklist`: verify BB `for poll in seq` loop isn't caught by `_watch_flag_blocklist`; add a `gh run watch` analogue only if a BB hang command exists. +- `backend/app/standards/deployment_standards.md`: add BB Pipelines section paralleling GitHub Actions; default artifacts gain `bitbucket-pipelines.yml`; note Server/DC has no managed CI. +- Tests: `build_deploy_recipe` golden per provider; `supports_managed_ci=False` produces notice + skips; `deploy_env` var; `deploy_extra_tools_for` gating. + +### Phase 6 — Onboarding / TUI / config / docs +- `mcp_server/tui/onboarding.py`: generalize hardcoded `_GITHUB` step into a provider-selecting step (GitHub / BB Cloud / BB Server) collecting `GIT_TOKEN`, namespace (org/workspace/project), `GIT_USER_NAME`, Server base URL. Mirror the LLM `_PROVIDER` pattern. +- `mcp_server/tui/config.py`: register new keys in `ENV_SECRET_KEYS`/`MASKED_KEYS` (load-time parity assertion is a hard gate). +- `.env.quickstart.example`, `QUICKSTART.md`, `specflow-init.sh` (`_GH_ORG` → provider-parameterized): add BB variants. +- MCP forwarding stays provider-agnostic (provider resolved server-side per workspace/key); forward a `GIT_PROVIDER` hint only if onboarding must set a per-key default. +- Tests: `mcp_server` config parity assertion; onboarding validation per provider. + +## Biggest Risks +1. **Server/DC has no managed CI** — graceful degradation, not a fake pipeline. Needs sign-off that this satisfies requirement #1 for Server/DC (see top-of-plan). +2. **Exact BB access-token clone/REST forms** differ Cloud vs Server (`x-token-auth`; `/scm/` path; `Bearer` REST). Isolated by the abstraction but need live validation against a real BB instance — `authenticated_clone_url` for Server is highest-uncertainty. +3. **Token-leak prevention is a security control** — apply all providers' sanitization patterns; Phase 0 golden tests must prove GitHub scrubbing unchanged; BB patterns (`x-token-auth:`, `ATCTT…`) must be added or BB tokens leak into logs/Slack. +4. **Field-rename back-compat** — `github_token_ciphertext` + `github-token` route are in production data/clients; read-fallback + route alias + one-release shim + idempotent non-destructive migration are mandatory. +5. **Deploy recipe is autonomous prompt text (up to 30 min)** — BB Cloud curl/jq poll loop must have the same hard timeout + non-fatal-403 semantics as GitHub and must not trip `_ci_blocklist`, else deploys hang mid-run with no human in the loop. + +## Verification +- `make unit-tests` after each phase; pass count ≥ baseline (record baseline first). New `backend/tests/unit/git_providers/` package: one module per provider per interface + registry/host-derivation test. Clone/deploy flow tests use in-memory DB + mocked `run_git`/`httpx` (no network). Do NOT use `make integration-tests` for these. +- `make check-complexity-diff` after Phases 2 and 5 (the two that refactor large existing files). +- End-to-end sanity: run `create_generation_session_repos.py --git-provider bitbucket_cloud --dry-run` to confirm provisioning wiring; exercise a BB Cloud workspace through clone→commit→push→archive with a real (or sandbox) BB access token; confirm the deploy recipe triggers a Pipelines custom run and polls to completion. Confirm GitHub path is byte-identical (golden tests green) — no regression. From 3343fdb72dc1161c7a30ffdb593d3cc6739a375b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Wr=C3=B3bel?= Date: Tue, 7 Jul 2026 11:50:17 +0200 Subject: [PATCH 2/2] Implement BitBucket Cloud support: add configuration options, provider resolution, and repository management. Introduce GitProvider abstraction for handling both GitHub and BitBucket, ensuring seamless integration and token sanitization across services. Update relevant tests and documentation to reflect new functionality. --- .env.quickstart.example | 20 + backend/app/core/config.py | 10 + backend/app/core/github_platform_secrets.py | 42 +- backend/app/services/git_provider.py | 107 +++++ backend/app/services/github_auth.py | 13 +- backend/app/services/workspace_pool.py | 32 +- .../create_generation_session_repos.py | 446 ++++++++++++------ .../test_create_generation_session_repos.py | 110 ++++- backend/test/services/test_git_provider.py | 114 +++++ backend/test/services/test_github_auth.py | 44 ++ backend/test/services/test_workspace_pool.py | 24 + .../bitbucket-pr1-commit-and-read.md | 230 +++++++++ mcp_server/tests/test_tui_app.py | 50 +- mcp_server/tests/test_tui_onboarding.py | 156 ++++-- mcp_server/tui/app.py | 41 +- mcp_server/tui/config.py | 3 + mcp_server/tui/onboarding.py | 218 ++++++--- 17 files changed, 1358 insertions(+), 302 deletions(-) create mode 100644 backend/app/services/git_provider.py create mode 100644 backend/test/services/test_git_provider.py create mode 100644 docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md diff --git a/.env.quickstart.example b/.env.quickstart.example index ab79258..6fe92f6 100644 --- a/.env.quickstart.example +++ b/.env.quickstart.example @@ -2,12 +2,25 @@ # REQUIRED — fill all of these before running specflow-init.sh # ============================================================================= +# Git host — set ONE of GITHUB_TOKEN or BITBUCKET_TOKEN below; leave the other +# blank. A deployment is either all-GitHub or all-BitBucket, never mixed. +# # GITHUB_TOKEN # What: GitHub Personal Access Token used to create and manage disposable # workspace repositories where agents commit generated code. # How: Generate a classic PAT with `repo` + `workflow` scopes: # https://github.com/settings/tokens/new?scopes=repo,workflow GITHUB_TOKEN= +# +# BITBUCKET_TOKEN / BITBUCKET_WORKSPACE (alternative to GitHub) +# What: A BitBucket Cloud Repository/Workspace access token, plus the +# workspace slug that owns the disposable workspace repos. No +# username needed — access tokens always authenticate as the fixed +# actor `x-token-auth`. +# How: Create a Workspace access token with repository read/write/admin +# scopes in BitBucket: Workspace settings > Access tokens. +# BITBUCKET_TOKEN= +# BITBUCKET_WORKSPACE= # P10Y_API_KEY # What: API key for Compass by P10Y — a code-complexity scoring service. @@ -111,6 +124,13 @@ GIT_USER_EMAIL= # or organization name. GITHUB_ORG= +# GIT_PROVIDER +# What: Explicit active git host override ("github" or "bitbucket_cloud"). +# How: Leave blank — auto-inferred from whichever of GITHUB_TOKEN / +# BITBUCKET_TOKEN is set. Only set this if you ever have both +# configured at once (which otherwise fails fast as ambiguous). +# GIT_PROVIDER= + # GIT_COMMITTER_USER_EMAIL # What: Email for system-level git commits (archive, workspace reset). # This is NOT your personal commit email — it identifies automated diff --git a/backend/app/core/config.py b/backend/app/core/config.py index bf494ad..200b700 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -199,6 +199,15 @@ def _empty_str_to_none_int(cls, v: object) -> object: validation_alias=AliasChoices("GITHUB_TOKEN_DEFAULT", "GITHUB_TOKEN"), ) + # BitBucket Cloud — mutually exclusive with GitHub (see git_provider.resolve_active_git_provider). + BITBUCKET_TOKEN_DEFAULT: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("BITBUCKET_TOKEN_DEFAULT", "BITBUCKET_TOKEN"), + ) + BITBUCKET_WORKSPACE: Optional[str] = None + # Explicit active-provider override; else inferred from exactly one of the tokens above. + GIT_PROVIDER: Optional[str] = None + # For cloning and https auth GIT_USER_NAME_DEFAULT: Optional[str] = Field( default=None, @@ -221,6 +230,7 @@ def _empty_str_to_none_int(cls, v: object) -> object: K8S_SECRET_KEY_ENCRYPTION: str = "token-encryption-key" K8S_SECRET_KEY_GITHUB_DEFAULT: str = "github-token-default" K8S_SECRET_KEY_GIT_USER_DEFAULT: str = "git-user-default" + K8S_SECRET_KEY_BITBUCKET_DEFAULT: str = "bitbucket-token-default" # Database Configuration DATABASE_TYPE: DatabaseType = DatabaseType.MEMORY diff --git a/backend/app/core/github_platform_secrets.py b/backend/app/core/github_platform_secrets.py index e5073cd..a4c31a1 100644 --- a/backend/app/core/github_platform_secrets.py +++ b/backend/app/core/github_platform_secrets.py @@ -16,6 +16,12 @@ from cryptography.fernet import Fernet, InvalidToken from app.core.config import Settings +from app.services.git_provider import ( + GitProvider, + resolve_active_git_provider, + resolve_active_git_provider_from_flags, + strategy_for, +) logger = logging.getLogger(__name__) @@ -29,6 +35,8 @@ class GithubPlatformSecrets: _fernet: Fernet github_token_default: Optional[str] git_user_name_default: Optional[str] + bitbucket_token_default: Optional[str] = None + active_provider: GitProvider = GitProvider.GITHUB def encrypt_token(self, token: str) -> str: return self._fernet.encrypt(token.encode()).decode() @@ -39,6 +47,16 @@ def decrypt_token(self, ciphertext: str) -> str: except InvalidToken as e: raise ValueError("GitHub token decryption failed") from e + @property + def active_default_token(self) -> Optional[str]: + if self.active_provider == GitProvider.BITBUCKET_CLOUD: + return self.bitbucket_token_default + return self.github_token_default + + @property + def active_git_user_default(self) -> str: + return self.git_user_name_default or strategy_for(self.active_provider).default_git_user + def get_github_platform_secrets() -> GithubPlatformSecrets: if _secrets is None: @@ -60,6 +78,8 @@ def init_github_platform_secrets_for_tests( fernet_key: bytes, github_token_default: str | None = "test-default-token", git_user_name_default: str | None = "test-user", + bitbucket_token_default: str | None = None, + active_provider: GitProvider = GitProvider.GITHUB, ) -> None: """Initialize secrets for unit tests (no K8s / .env).""" global _secrets @@ -67,6 +87,8 @@ def init_github_platform_secrets_for_tests( _fernet=Fernet(fernet_key), github_token_default=github_token_default, git_user_name_default=git_user_name_default, + bitbucket_token_default=bitbucket_token_default, + active_provider=active_provider, ) @@ -97,6 +119,8 @@ def _build_secrets_from_map( encryption_key_field: str, github_default_field: str, git_user_field: str, + bitbucket_default_field: str, + git_provider_override: str | None, ) -> GithubPlatformSecrets: enc_raw = data.get(encryption_key_field) if not enc_raw: @@ -106,10 +130,20 @@ def _build_secrets_from_map( fernet = Fernet(_decode_secret_value(enc_raw).strip().encode()) gh = data.get(github_default_field) gu = data.get(git_user_field) + bb = data.get(bitbucket_default_field) + gh_value = gh.strip() if gh else None + bb_value = bb.strip() if bb else None + active_provider = resolve_active_git_provider_from_flags( + override=git_provider_override, + has_github_token=bool(gh_value), + has_bitbucket_token=bool(bb_value), + ) return GithubPlatformSecrets( _fernet=fernet, - github_token_default=gh.strip() if gh else None, + github_token_default=gh_value, git_user_name_default=gu.strip() if gu else None, + bitbucket_token_default=bb_value, + active_provider=active_provider, ) @@ -122,10 +156,14 @@ def _load_from_env(settings: Settings) -> GithubPlatformSecrets: fernet = Fernet(str(enc).strip().encode()) gh_default = settings.GITHUB_TOKEN_DEFAULT git_user = settings.GIT_USER_NAME_DEFAULT + bb_default = settings.BITBUCKET_TOKEN_DEFAULT + active_provider = resolve_active_git_provider(settings) return GithubPlatformSecrets( _fernet=fernet, github_token_default=gh_default.strip() if gh_default else None, git_user_name_default=git_user.strip() if git_user else None, + bitbucket_token_default=bb_default.strip() if bb_default else None, + active_provider=active_provider, ) @@ -149,6 +187,8 @@ def init_github_platform_secrets(settings: Settings) -> None: encryption_key_field=settings.K8S_SECRET_KEY_ENCRYPTION, github_default_field=settings.K8S_SECRET_KEY_GITHUB_DEFAULT, git_user_field=settings.K8S_SECRET_KEY_GIT_USER_DEFAULT, + bitbucket_default_field=settings.K8S_SECRET_KEY_BITBUCKET_DEFAULT, + git_provider_override=settings.GIT_PROVIDER, ) logger.info( "Loaded GitHub platform secrets from Kubernetes API (namespace=%s, name=%s)", diff --git a/backend/app/services/git_provider.py b/backend/app/services/git_provider.py new file mode 100644 index 0000000..f497d6e --- /dev/null +++ b/backend/app/services/git_provider.py @@ -0,0 +1,107 @@ +""" +Minimal git-host provider abstraction. + +A deployment is either all-GitHub or all-BitBucket, never mixed — the active +provider is resolved once, globally, from configured tokens (or an explicit +override). See docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from app.core.config import Settings + + +class GitProvider(str, Enum): + GITHUB = "github" + BITBUCKET_CLOUD = "bitbucket_cloud" + + +class GitProviderResolutionError(Exception): + """Raised when the active git provider cannot be determined unambiguously.""" + + +@dataclass(frozen=True) +class GitHostStrategy: + provider: GitProvider + default_git_user: str + sanitization_patterns: tuple[tuple[re.Pattern[str], str], ...] + + +_GITHUB = GitHostStrategy( + provider=GitProvider.GITHUB, + default_git_user="x-access-token", + sanitization_patterns=( + (re.compile(r"(https?://[^:]+:)[^@]+(@github\.com)"), r"\1[REDACTED]\2"), + (re.compile(r"gh[ps]_[A-Za-z0-9_]+"), "[REDACTED]"), + (re.compile(r"github_pat_[A-Za-z0-9_]+"), "[REDACTED]"), + ), +) + +_BITBUCKET = GitHostStrategy( + provider=GitProvider.BITBUCKET_CLOUD, + default_git_user="x-token-auth", + sanitization_patterns=( + (re.compile(r"(https?://[^:]+:)[^@]+(@bitbucket\.org)"), r"\1[REDACTED]\2"), + (re.compile(r"ATCTT[A-Za-z0-9_=\-]+"), "[REDACTED]"), + ), +) + +_STRATEGIES: dict[GitProvider, GitHostStrategy] = { + GitProvider.GITHUB: _GITHUB, + GitProvider.BITBUCKET_CLOUD: _BITBUCKET, +} + + +def strategy_for(provider: GitProvider) -> GitHostStrategy: + return _STRATEGIES[provider] + + +def all_strategies() -> list[GitHostStrategy]: + return list(_STRATEGIES.values()) + + +def resolve_active_git_provider_from_flags( + *, + override: Optional[str], + has_github_token: bool, + has_bitbucket_token: bool, +) -> GitProvider: + """Pure resolution logic shared by settings-based and Kubernetes-secret-map loading.""" + explicit = (override or "").strip().lower() + if explicit: + try: + return GitProvider(explicit) + except ValueError as e: + valid = [p.value for p in GitProvider] + raise GitProviderResolutionError( + f"GIT_PROVIDER={explicit!r} is not valid; expected one of {valid}" + ) from e + + if has_github_token and has_bitbucket_token: + raise GitProviderResolutionError( + "Both a GitHub and a BitBucket token are configured; set GIT_PROVIDER " + "explicitly to choose the active git host (mixing providers is not supported)" + ) + if has_github_token: + return GitProvider.GITHUB + if has_bitbucket_token: + return GitProvider.BITBUCKET_CLOUD + + raise GitProviderResolutionError( + "No git provider configured; set GITHUB_TOKEN or BITBUCKET_TOKEN " + "(and GIT_PROVIDER if both might be set)" + ) + + +def resolve_active_git_provider(settings: "Settings") -> GitProvider: + return resolve_active_git_provider_from_flags( + override=settings.GIT_PROVIDER, + has_github_token=bool(settings.GITHUB_TOKEN_DEFAULT), + has_bitbucket_token=bool(settings.BITBUCKET_TOKEN_DEFAULT), + ) diff --git a/backend/app/services/github_auth.py b/backend/app/services/github_auth.py index ed8beb3..ea10638 100644 --- a/backend/app/services/github_auth.py +++ b/backend/app/services/github_auth.py @@ -44,22 +44,23 @@ def resolve_github_auth_for_api_key_document( token = secrets.decrypt_token(str(ciphertext)) except ValueError as e: raise GithubAuthResolutionError("Stored GitHub token could not be decrypted") from e - user = (per_key_user or secrets.git_user_name_default or "x-access-token").strip() + user = (per_key_user or secrets.active_git_user_default).strip() return GithubAuthContext(git_user_name=user, token=token) if pool == DEFAULT_WORKSPACE_POOL: - if not secrets.github_token_default or not secrets.git_user_name_default: + if not secrets.active_default_token or not secrets.git_user_name_default: raise GithubAuthResolutionError( - "Default workspace pool requires GITHUB_TOKEN_DEFAULT and GIT_USER_NAME_DEFAULT " - "(platform secrets) when no per-key token is stored" + f"Default workspace pool requires a default token and GIT_USER_NAME_DEFAULT " + f"(platform secrets) for the active provider ({secrets.active_provider.value}) " + f"when no per-key token is stored" ) return GithubAuthContext( git_user_name=secrets.git_user_name_default, - token=secrets.github_token_default, + token=secrets.active_default_token, ) raise GithubAuthResolutionError( - f"Workspace pool {pool!r} requires a GitHub PAT: call PUT /api/v1/auth/github-token " + f"Workspace pool {pool!r} requires a git PAT: call PUT /api/v1/auth/github-token " f"with this API key before running git operations" ) diff --git a/backend/app/services/workspace_pool.py b/backend/app/services/workspace_pool.py index 51c4ad4..af32992 100644 --- a/backend/app/services/workspace_pool.py +++ b/backend/app/services/workspace_pool.py @@ -16,7 +16,6 @@ import logging import os from pathlib import Path -import re import shutil from typing import Any, Dict, List, Optional @@ -32,6 +31,7 @@ GithubAuthResolutionError, resolve_github_auth_for_generation_id, ) +from app.services.git_provider import all_strategies from app.database.interface import IDatabase from app.state import WorkspaceStateMachine from app.state.db_adapter import StateMachineDBAdapter @@ -173,31 +173,17 @@ def _sanitize_token_in_message(message: str, token: Optional[str] = None) -> str return message sanitized = message - + # If specific token provided, replace it if token: sanitized = sanitized.replace(token, "[REDACTED]") - - # Also sanitize common patterns for GitHub tokens in URLs - # Pattern: https://username:token@github.com/... - sanitized = re.sub( - r'(https?://[^:]+:)[^@]+(@github\.com)', - r'\1[REDACTED]\2', - sanitized - ) - - # Pattern: ghp_XXXX or github_pat_XXXX tokens (GitHub personal access tokens) - sanitized = re.sub( - r'gh[ps]_[A-Za-z0-9_]+', - '[REDACTED]', - sanitized - ) - sanitized = re.sub( - r'github_pat_[A-Za-z0-9_]+', - '[REDACTED]', - sanitized - ) - + + # Sanitize known token/URL patterns for every provider, not just the active one — + # cheap, and defends even if a token from another provider leaks into these logs. + for strategy in all_strategies(): + for pattern, replacement in strategy.sanitization_patterns: + sanitized = pattern.sub(replacement, sanitized) + return sanitized def __init__(self, db: IDatabase, workspace_base_path: Optional[str | Path] = None): diff --git a/backend/scripts/create_generation_session_repos.py b/backend/scripts/create_generation_session_repos.py index f227eec..c7a506d 100755 --- a/backend/scripts/create_generation_session_repos.py +++ b/backend/scripts/create_generation_session_repos.py @@ -42,7 +42,7 @@ from pathlib import Path import sys import time -from typing import Any, Dict, List, Optional +from typing import Any, Awaitable, Callable, Dict, List, Optional from dotenv import load_dotenv import httpx @@ -66,6 +66,12 @@ from app.database.factory import get_database # noqa: E402 from app.database.firestore import FirestoreDatabase # noqa: E402 from app.database.interface import IDatabase # noqa: E402 +from app.services.git_provider import ( # noqa: E402 + GitProvider, + GitProviderResolutionError, + resolve_active_git_provider_from_flags, + strategy_for, +) from app.services.p10y.p10y_api_client import P10YInternalAPIClient # noqa: E402 LIVE_REPOSITORY_STATUS = "Live" @@ -202,93 +208,197 @@ async def close(self): await self.client.aclose() -async def create_github_repositories( - github_client: GitHubAPIClient, +class BitbucketCloudAPIClient: + """Client for BitBucket Cloud REST API repository operations.""" + + def __init__(self, access_token: str, workspace: str): + """ + Args: + access_token: BitBucket Cloud Repository/Workspace access token + workspace: BitBucket workspace slug (owner of repos) + """ + self.workspace = workspace + self.base_url = "https://api.bitbucket.org/2.0" + self.headers = {"Authorization": f"Bearer {access_token}"} + self.client = httpx.AsyncClient(timeout=30.0) + + async def create_repository(self, repo_slug: str) -> Dict[str, Any]: + """Create a private repo. Treats an already-exists 400 as idempotent success.""" + url = f"{self.base_url}/repositories/{self.workspace}/{repo_slug}" + response = await self.client.post( + url, json={"scm": "git", "is_private": True}, headers=self.headers + ) + if response.status_code == 400 and "already exists" in response.text.lower(): + return await self.get_repository(repo_slug) + response.raise_for_status() + return response.json() + + async def get_repository(self, repo_slug: str) -> Dict[str, Any]: + url = f"{self.base_url}/repositories/{self.workspace}/{repo_slug}" + response = await self.client.get(url, headers=self.headers) + response.raise_for_status() + return response.json() + + async def repository_exists(self, repo_slug: str) -> bool: + url = f"{self.base_url}/repositories/{self.workspace}/{repo_slug}" + response = await self.client.get(url, headers=self.headers) + return response.status_code == 200 + + async def close(self): + """Close the HTTP client.""" + await self.client.aclose() + + +def _repo_url(provider: GitProvider, owner: str, repo_name: str) -> str: + if provider == GitProvider.BITBUCKET_CLOUD: + return f"https://bitbucket.org/{owner}/{repo_name}" + return f"https://github.com/{owner}/{repo_name}" + + +async def _create_repositories( + client: "GitHubAPIClient | BitbucketCloudAPIClient", + provider: GitProvider, + owner: str, prefix: str, start_num: int, end_num: int, - team_slug: str | None, - delay: float = 0.1, + delay: float, + on_created: Optional[Callable[[str], Awaitable[None]]] = None, ) -> List[Dict[str, Any]]: - """ - Create multiple GitHub repositories with sequential numbering under the client's org. + """Shared create-or-skip-if-exists loop for both providers. - Args: - github_client: Initialized GitHub API client (org is github_client.org) - prefix: Repository name prefix (e.g., "generation-workspace") - start_num: Starting number (inclusive) - end_num: Ending number (inclusive) - team_slug: If set, grant this team Write access (push) on each repo - delay: Delay between requests in seconds (default: 0.1) - - Returns: - List of created repository data + ``on_created`` runs after each repo (create or already-exists) — used for + GitHub's per-repo team-grant step; BitBucket has no equivalent. """ - created_repos = [] - org = github_client.org + created_repos: List[Dict[str, Any]] = [] for num in range(start_num, end_num + 1): repo_name = f"{prefix}{num}" - # Check if repo already exists - if await github_client.repository_exists(repo_name): + if await client.repository_exists(repo_name): print(f"⚠️ Repository '{repo_name}' already exists, skipping creation") - # Still add to list for P10y sync created_repos.append({ "name": repo_name, - "full_name": f"{org}/{repo_name}", - "html_url": f"https://github.com/{org}/{repo_name}", + "full_name": f"{owner}/{repo_name}", + "html_url": _repo_url(provider, owner, repo_name), "already_existed": True, }) else: try: - print(f"📦 Creating repository: {org}/{repo_name}") - repo_data = await github_client.create_repository(repo_name) + print(f"📦 Creating repository: {owner}/{repo_name}") + repo_data = await client.create_repository(repo_name) created_repos.append(repo_data) - print(f"✅ Created: {repo_data['html_url']}") + print(f"✅ Created: {_repo_url(provider, owner, repo_name)}") except httpx.HTTPStatusError as e: print(f"❌ Failed to create {repo_name}: {e}") print(f" Response: {e.response.text}") raise - if team_slug: - try: - await github_client.add_team_repository_write(team_slug, repo_name) - print(f" 👥 Team '{team_slug}' granted Write on {repo_name}") - except httpx.HTTPStatusError as e: - print(f"❌ Failed to add team {team_slug} to {repo_name}: {e}") - print(f" Response: {e.response.text}") - raise + if on_created is not None: + await on_created(repo_name) - # Add delay to avoid rate limiting if num < end_num: await asyncio.sleep(delay) return created_repos +async def create_bitbucket_repositories( + bitbucket_client: BitbucketCloudAPIClient, + prefix: str, + start_num: int, + end_num: int, + delay: float = 0.1, +) -> List[Dict[str, Any]]: + """Create multiple BitBucket Cloud repositories with sequential numbering. + + No team-grant step: the access token's owner already has write access to + repos it creates in the workspace. + """ + return await _create_repositories( + bitbucket_client, + GitProvider.BITBUCKET_CLOUD, + bitbucket_client.workspace, + prefix, + start_num, + end_num, + delay, + ) + + +async def create_github_repositories( + github_client: GitHubAPIClient, + prefix: str, + start_num: int, + end_num: int, + team_slug: str | None, + delay: float = 0.1, +) -> List[Dict[str, Any]]: + """ + Create multiple GitHub repositories with sequential numbering under the client's org. + + Args: + github_client: Initialized GitHub API client (org is github_client.org) + prefix: Repository name prefix (e.g., "generation-workspace") + start_num: Starting number (inclusive) + end_num: Ending number (inclusive) + team_slug: If set, grant this team Write access (push) on each repo + delay: Delay between requests in seconds (default: 0.1) + + Returns: + List of created repository data + """ + def _team_grant_hook(slug: str) -> Callable[[str], Awaitable[None]]: + async def _grant_team(repo_name: str) -> None: + try: + await github_client.add_team_repository_write(slug, repo_name) + print(f" 👥 Team '{slug}' granted Write on {repo_name}") + except httpx.HTTPStatusError as e: + print(f"❌ Failed to add team {slug} to {repo_name}: {e}") + print(f" Response: {e.response.text}") + raise + + return _grant_team + + return await _create_repositories( + github_client, + GitProvider.GITHUB, + github_client.org, + prefix, + start_num, + end_num, + delay, + on_created=_team_grant_hook(team_slug) if team_slug else None, + ) + + def print_dry_run_plan( *, + provider: GitProvider, token_login: str, - github_org: str, - team_slug: str | None, + owner: str, + team_slug: Optional[str], skip_team: bool, skip_github: bool, prefix: str, start_num: int, end_num: int, ) -> None: - """Print planned GitHub org, team, token actor, and full repo URLs (no API calls).""" + """Print planned provider owner, team (GitHub only), token actor, and full repo URLs (no API calls).""" + is_github = provider == GitProvider.GITHUB print("\n" + "=" * 80) - print("🔍 DRY RUN — GitHub stage only (no repos created, no team grants, no P10y/Firestore)") + print(f"🔍 DRY RUN — {provider.value} stage only (no repos created, no team grants, no P10y/Firestore)") print("=" * 80) print( " Token authenticates as: " f"{token_login}\n" - " (This identity must have permission to create repos in the org and manage team access.)" + " (This identity must have permission to create repos in the org/workspace" + + (" and manage team access." if is_github else ".") ) - print(f" Organization (repo owner): {github_org}") - if skip_github: + print(f" Organization/workspace (repo owner): {owner}") + if not is_github: + print(" Team Write (push): — (n/a for BitBucket; token owner already has write)") + elif skip_github: print(" Team Write (push): — (--skip-github; would not run GitHub API)") elif skip_team: print(" Team Write (push): — (--skip-team)") @@ -300,19 +410,24 @@ def print_dry_run_plan( print(" Repository full URLs (same as Firestore repo_url):") for num in range(start_num, end_num + 1): repo_name = f"{prefix}{num}" - print(f" https://github.com/{github_org}/{repo_name}") + print(f" {_repo_url(provider, owner, repo_name)}") print() if skip_github: - print(" Would skip: POST /orgs/{org}/repos (no new repositories)") - else: - print(f" Would create {end_num - start_num + 1} private repos: POST /orgs/{github_org}/repos") + print(" Would skip: repository creation API call (no new repositories)") + elif is_github: + print(f" Would create {end_num - start_num + 1} private repos: POST /orgs/{owner}/repos") if not skip_team and team_slug: print( f" Would grant team '{team_slug}' Write on each: " - f"PUT /orgs/{github_org}/teams/{team_slug}/repos/{github_org}/" + f"PUT /orgs/{owner}/teams/{team_slug}/repos/{owner}/" ) elif skip_team: print(" Would skip: team repository permission updates (--skip-team)") + else: + print( + f" Would create {end_num - start_num + 1} private repos: " + f"POST /repositories/{owner}/ (BitBucket Cloud)" + ) print("=" * 80) @@ -603,7 +718,7 @@ def create_workspace_document( workspace_id: str, set_number: int, repo_url: str, - p10y_repository_id: int, + p10y_repository_id: Optional[int], now: datetime, workspace_pool: str = "default" ) -> dict: @@ -651,13 +766,14 @@ def create_workspace_document( async def add_workspaces_to_firestore( - repo_id_map: Dict[str, int], - github_org: str, + repo_id_map: Dict[str, Optional[int]], + owner: str, prefix: str, start_num: int, workspace_pool: str = "default", firestore_project_id: Optional[str] = None, firestore_database_id: Optional[str] = None, + provider: GitProvider = GitProvider.GITHUB, ) -> None: """ Add workspace entries to Firestore database. @@ -704,7 +820,7 @@ async def add_workspaces_to_firestore( workspace_id = f"ws-{set_number:02d}-{workspace_index}" # Create repo URL - repo_url = f"https://github.com/{github_org}/{repo_name}" + repo_url = _repo_url(provider, owner, repo_name) # Create workspace document workspace_doc = create_workspace_document( @@ -743,26 +859,26 @@ async def add_workspaces_to_firestore( def emit_workspace_config( - repo_id_map: Dict[str, int], - github_org: str, + repo_id_map: Dict[str, Optional[int]], + owner: str, prefix: str, workspace_pool: str, output_path: str, ordered_repos: Optional[List[str]] = None, + provider: GitProvider = GitProvider.GITHUB, ) -> None: """ Write a JSON workspace-config file in the exact schema consumed by ``init_firestore.py --workspace-config``: [{"workspace_id": str, "repo_url": str, - "p10y_repository_id": int, "workspace_pool": str}, ...] - - All four fields are required (matching WorkspaceConfig dataclass). + "p10y_repository_id": int | None, "workspace_pool": str}, ...] Field sources: - workspace_id: derived the same way as add_workspaces_to_firestore (ws-{set:02d}-{idx}) - - repo_url: https://github.com/{github_org}/{repo_name} - - p10y_repository_id: integer value from repo_id_map[repo_name] + - repo_url: built from ``provider`` (github.com or bitbucket.org) + owner + repo_name + - p10y_repository_id: value from repo_id_map[repo_name]; null when P10Y lookup was + skipped (e.g. BitBucket provisioning — registered externally once Compass connects it) - workspace_pool: workspace_pool argument When ordered_repos is provided (the --repos path), workspace IDs are assigned @@ -771,10 +887,11 @@ def emit_workspace_config( def _make_entry(repo_name: str, idx: int) -> Dict[str, Any]: set_number = (idx // 3) + 1 workspace_index = (idx % 3) + 1 + p10y_id = repo_id_map[repo_name] return { "workspace_id": f"ws-{set_number:02d}-{workspace_index}", - "repo_url": f"https://github.com/{github_org}/{repo_name}", - "p10y_repository_id": int(repo_id_map[repo_name]), + "repo_url": _repo_url(provider, owner, repo_name), + "p10y_repository_id": int(p10y_id) if p10y_id is not None else None, "workspace_pool": workspace_pool, } @@ -851,6 +968,24 @@ async def main(): metavar="ORG", help="GitHub organization login; repos are ORG/{PREFIX}N. Env: GITHUB_ORG, GITHUB_ORG_DEFAULT", ) + parser.add_argument( + "--git-provider", + type=str, + choices=[p.value for p in GitProvider], + default=None, + help=( + "Active git host. Default: inferred from whichever of GITHUB_TOKEN / " + "BITBUCKET_TOKEN is configured (error if both or neither, unless GIT_PROVIDER " + "is set in the environment)." + ), + ) + parser.add_argument( + "--bitbucket-workspace", + type=str, + default=os.getenv("BITBUCKET_WORKSPACE"), + metavar="WORKSPACE", + help="BitBucket Cloud workspace slug; repos are WORKSPACE/{PREFIX}N. Env: BITBUCKET_WORKSPACE", + ) parser.add_argument( "--team", type=str, @@ -946,37 +1081,47 @@ async def main(): workspace_pool = args.workspace_pool.lower() if args.workspace_pool else "default" github_org = (args.github_org or "").strip() or None + bitbucket_workspace = (args.bitbucket_workspace or "").strip() or None team_slug = None if args.skip_team else ((args.team or "").strip() or None) gcp_cli = (args.gcp_project or "").strip() or None fsdb_cli = (args.firestore_database or "").strip() or None firestore_target_from_cli = bool(gcp_cli and fsdb_cli) + try: + provider = resolve_active_git_provider_from_flags( + override=args.git_provider or cfg.GIT_PROVIDER, + has_github_token=bool(cfg.GITHUB_TOKEN_DEFAULT), + has_bitbucket_token=bool(cfg.BITBUCKET_TOKEN_DEFAULT), + ) + except GitProviderResolutionError as e: + print(f"❌ Error: {e}") + sys.exit(1) + + is_bitbucket = provider == GitProvider.BITBUCKET_CLOUD + if is_bitbucket: + # No Compass/GitHub registration step for BitBucket in this PR (see plan doc); + # p10y_repository_id is set externally once Compass connects the repo. + args.skip_metrics = True + owner = bitbucket_workspace if is_bitbucket else github_org + owner_flag = "--bitbucket-workspace (or BITBUCKET_WORKSPACE)" if is_bitbucket else "--github-org (or GITHUB_ORG / GITHUB_ORG_DEFAULT)" + # Validate arguments if args.start > args.end: print("❌ Error: start number must be less than or equal to end number") sys.exit(1) - if args.dry_run and not github_org: - print( - "❌ Error: --dry-run requires --github-org (or GITHUB_ORG / GITHUB_ORG_DEFAULT) " - "to list repository URLs" - ) + if args.dry_run and not owner: + print(f"❌ Error: --dry-run requires {owner_flag} to list repository URLs") sys.exit(1) if not args.skip_github: - if not github_org: - print( - "❌ Error: --github-org is required to create repositories " - "(or set GITHUB_ORG / GITHUB_ORG_DEFAULT)" - ) + if not owner: + print(f"❌ Error: {owner_flag} is required to create repositories") sys.exit(1) - if not args.dry_run and not args.skip_firestore and not github_org: - print( - "❌ Error: --github-org is required for Firestore repo URLs " - "(or set GITHUB_ORG / GITHUB_ORG_DEFAULT)" - ) + if not args.dry_run and not args.skip_firestore and not owner: + print(f"❌ Error: {owner_flag} is required for Firestore repo URLs") sys.exit(1) if not args.skip_firestore and not args.dry_run: @@ -998,16 +1143,19 @@ async def main(): # Check required environment variables github_token = cfg.GITHUB_TOKEN_DEFAULT + bitbucket_token = cfg.BITBUCKET_TOKEN_DEFAULT + token = bitbucket_token if is_bitbucket else github_token + token_env_flag = "BITBUCKET_TOKEN" if is_bitbucket else "GITHUB_TOKEN_DEFAULT (or legacy GITHUB_TOKEN)" p10y_api_key = cfg.P10Y_API_KEY # gitleaks:allow - variable assignment, not a literal p10y_base_url = cfg.P10Y_BASE_URL p10y_org_id = cfg.P10Y_ORGANISATION_ID git_username = cfg.GIT_USER_NAME_DEFAULT - if not args.dry_run and not github_token: - print("❌ Error: GITHUB_TOKEN_DEFAULT (or legacy GITHUB_TOKEN) not set in environment") + if not args.dry_run and not token: + print(f"❌ Error: {token_env_flag} not set in environment") sys.exit(1) - if not args.dry_run: + if not args.dry_run and not is_bitbucket: if not p10y_api_key: print("❌ Error: P10Y_API_KEY not set in environment") sys.exit(1) @@ -1016,14 +1164,14 @@ async def main(): print("❌ Error: P10Y_ORGANISATION_ID not set in environment") sys.exit(1) - if args.dry_run and not github_token: - print( - "⚠️ GITHUB_TOKEN_DEFAULT not set — resolve token actor via " - "GIT_USER_NAME_DEFAULT or add a token for GET /user" - ) + if args.dry_run and not token: + print(f"⚠️ {token_env_flag} not set — resolve token actor via GIT_USER_NAME_DEFAULT or add a token") + if is_bitbucket: + # BitBucket access tokens always authenticate as a fixed actor — no username to resolve. + git_username = strategy_for(provider).default_git_user # Optional: resolve token owner's login for logging (org repos do not use this as owner) - if not git_username and github_token: + elif not git_username and github_token: print("⚠️ GIT_USER_NAME_DEFAULT not set, fetching token owner from GitHub API...") try: temp_client = GitHubAPIClient(github_token, "_") @@ -1056,10 +1204,12 @@ async def main(): print(f" Prefix: {args.prefix}") print(f" Range: {args.start} to {args.end}") print(f" Count: {args.end - args.start + 1} repositories") - print(f" GitHub org (repo owner): {github_org or '—'}") - print(f" Team Write (slug): {team_slug or '—'}") + print(f" Git provider: {provider.value}") + print(f" Organization/workspace (repo owner): {owner or '—'}") + if not is_bitbucket: + print(f" Team Write (slug): {team_slug or '—'}") print(f" Token login (info): {git_username}") - if not args.dry_run: + if not args.dry_run and not is_bitbucket: print(f" P10y Org ID: {p10y_org_id}") print(f" Workspace Pool: {workspace_pool}") if firestore_target_from_cli: @@ -1074,8 +1224,9 @@ async def main(): if args.dry_run: print_dry_run_plan( + provider=provider, token_login=git_username, - github_org=github_org or "", + owner=owner or "", team_slug=team_slug, skip_team=args.skip_team, skip_github=args.skip_github, @@ -1083,75 +1234,95 @@ async def main(): start_num=args.start, end_num=args.end, ) - print("\n✅ Dry run finished — exited before GitHub API calls, P10y, and Firestore.") + print("\n✅ Dry run finished — exited before any provider API calls, P10y, and Firestore.") return - github_client = GitHubAPIClient(github_token, github_org or "_") + provider_client: GitHubAPIClient | BitbucketCloudAPIClient + if is_bitbucket: + provider_client = BitbucketCloudAPIClient(token, owner or "_") + else: + provider_client = GitHubAPIClient(token, owner or "_") p10y_client = P10YInternalAPIClient(base_url=p10y_base_url, api_key=p10y_api_key) - + try: - # Step 1: Create GitHub repositories + # Step 1: Create repositories if own_repo_list is not None: # --repos path: repos already exist, skip creation entirely - print("\n⏭️ Skipping GitHub repository creation (--repos provided)") + print("\n⏭️ Skipping repository creation (--repos provided)") created_repos = [] repo_names = own_repo_list - elif not args.skip_github: - created_repos = await create_github_repositories( - github_client, - args.prefix, - args.start, - args.end, - team_slug, - args.delay, - ) - print(f"\n✅ Created/found {len(created_repos)} repositories") - repo_names = [f"{args.prefix}{num}" for num in range(args.start, args.end + 1)] else: - print("\n⏭️ Skipping GitHub repository creation") - created_repos = [] repo_names = [f"{args.prefix}{num}" for num in range(args.start, args.end + 1)] + if not args.skip_github: + if is_bitbucket: + created_repos = await create_bitbucket_repositories( + provider_client, + args.prefix, + args.start, + args.end, + args.delay, + ) + else: + created_repos = await create_github_repositories( + provider_client, + args.prefix, + args.start, + args.end, + team_slug, + args.delay, + ) + print(f"\n✅ Created/found {len(created_repos)} repositories") + else: + print("\n⏭️ Skipping repository creation") + created_repos = [] + + if is_bitbucket: + # No Compass/P10Y registration in this PR — external setup connects the repo + # later, and p10y_repository_id stays null until it does. + print("\n⏭️ Skipping P10Y repository ID lookup (BitBucket provisioning)") + repo_id_map: Dict[str, Optional[int]] = {name: None for name in repo_names} + else: + # Step 2: Get P10y repository IDs + # For --repos, search with empty prefix to match arbitrary names across the full org. + p10y_search_prefix = "" if own_repo_list is not None else args.prefix + repo_id_map = await get_repository_ids( + p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org + ) - # Step 2: Get P10y repository IDs - # For --repos, search with empty prefix to match arbitrary names across the full org. - p10y_search_prefix = "" if own_repo_list is not None else args.prefix - repo_id_map = await get_repository_ids( - p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org - ) + # Newly created GitHub repos are invisible to P10Y until Compass re-fetches the + # connection that owns them. When some are missing, trigger a re-fetch, wait, and + # look up again (twice). Failing to do this is what let an expansion run (K=1 → K=3) + # silently write a too-small workspaces.json and under-seed Firestore. + missing = [r for r in repo_names if r not in repo_id_map] + if missing: + print(f"\n🔄 {len(missing)} repo(s) not in P10Y yet: {', '.join(missing)} — triggering re-fetch ...") + await trigger_repository_refetch(p10y_client, p10y_org_id, github_org, repo_names) - # Newly created GitHub repos are invisible to P10Y until Compass re-fetches the - # connection that owns them. When some are missing, trigger a re-fetch, wait, and - # look up again (twice). Failing to do this is what let an expansion run (K=1 → K=3) - # silently write a too-small workspaces.json and under-seed Firestore. - missing = [r for r in repo_names if r not in repo_id_map] - if missing: - print(f"\n🔄 {len(missing)} repo(s) not in P10Y yet: {', '.join(missing)} — triggering re-fetch ...") - await trigger_repository_refetch(p10y_client, p10y_org_id, github_org, repo_names) - - deadline = time.time() + P10Y_REFETCH_TIMEOUT_SECONDS - while missing and time.time() < deadline: - print(f" ⏱️ {len(missing)} still missing; re-checking in {P10Y_REFETCH_POLL_SECONDS}s ...") - await asyncio.sleep(P10Y_REFETCH_POLL_SECONDS) - repo_id_map = await get_repository_ids( - p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org - ) - missing = [r for r in repo_names if r not in repo_id_map] + deadline = time.time() + P10Y_REFETCH_TIMEOUT_SECONDS + while missing and time.time() < deadline: + print(f" ⏱️ {len(missing)} still missing; re-checking in {P10Y_REFETCH_POLL_SECONDS}s ...") + await asyncio.sleep(P10Y_REFETCH_POLL_SECONDS) + repo_id_map = await get_repository_ids( + p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org + ) + missing = [r for r in repo_names if r not in repo_id_map] - if missing: - print(f"\n❌ Could not resolve P10Y IDs for {len(missing)} repo(s): {', '.join(missing)}.\nExiting the script after {P10Y_REFETCH_TIMEOUT_SECONDS} seconds. Potential debugging: Verify if on P10Y UI repo list, try re-fetching them manually, verify Integration to Github") - sys.exit(1) + if missing: + print(f"\n❌ Could not resolve P10Y IDs for {len(missing)} repo(s): {', '.join(missing)}.\nExiting the script after {P10Y_REFETCH_TIMEOUT_SECONDS} seconds. Potential debugging: Verify if on P10Y UI repo list, try re-fetching them manually, verify Integration to Github") + sys.exit(1) - repo_ids = list(repo_id_map.values()) + repo_ids = [rid for rid in repo_id_map.values() if rid is not None] # Emit workspace config JSON if requested (schema matches init_firestore.py --workspace-config) if args.output_workspace_config: emit_workspace_config( repo_id_map=repo_id_map, - github_org=github_org or "", + owner=owner or "", prefix=args.prefix, workspace_pool=workspace_pool, output_path=args.output_workspace_config, ordered_repos=own_repo_list, + provider=provider, ) # Step 4: Start metrics calculation only for repos that are not already Live. @@ -1186,12 +1357,13 @@ async def main(): if not args.skip_firestore and own_repo_list is None: await add_workspaces_to_firestore( repo_id_map, - github_org, + owner, args.prefix, args.start, workspace_pool, firestore_project_id=gcp_cli if firestore_target_from_cli else None, firestore_database_id=fsdb_cli if firestore_target_from_cli else None, + provider=provider, ) else: print("\n⏭️ Skipping Firestore workspace creation") @@ -1216,7 +1388,7 @@ async def main(): traceback.print_exc() sys.exit(1) finally: - await github_client.close() + await provider_client.close() await p10y_client.close() print("\n✅ Script completed successfully!") diff --git a/backend/test/scripts/test_create_generation_session_repos.py b/backend/test/scripts/test_create_generation_session_repos.py index 50a0368..b3b2d27 100644 --- a/backend/test/scripts/test_create_generation_session_repos.py +++ b/backend/test/scripts/test_create_generation_session_repos.py @@ -16,11 +16,13 @@ import json import re from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import scripts.create_generation_session_repos as cgsr +from app.services.git_provider import GitProvider from scripts.init_firestore import WorkspaceConfig # --------------------------------------------------------------------------- @@ -48,7 +50,7 @@ def test_roundtrip_through_workspace_config(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="test-org", + owner="test-org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -72,7 +74,7 @@ def test_all_four_required_fields_present(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="my-org", + owner="my-org", prefix="specflow-workspace", workspace_pool="testpool", output_path=output, @@ -95,7 +97,7 @@ def test_workspace_id_derivation(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -113,7 +115,7 @@ def test_repo_url_uses_github_org(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="my-org", + owner="my-org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -129,7 +131,7 @@ def test_p10y_repository_id_is_int_not_bool(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -147,7 +149,7 @@ def test_workspace_pool_propagated(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="custom-pool", output_path=output, @@ -162,7 +164,7 @@ def test_output_file_created_in_subdir(self, tmp_path): cgsr.emit_workspace_config( repo_id_map={"specflow-workspace1": 1}, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -444,3 +446,95 @@ def test_script_file_exists_on_disk(self): assert _SCRIPT_FILE.exists(), ( f"Script not found at expected path: {_SCRIPT_FILE}" ) + + +# =========================================================================== +# (f) _repo_url — provider-generic URL construction +# =========================================================================== + +class TestRepoUrl: + def test_github(self): + assert cgsr._repo_url(GitProvider.GITHUB, "my-org", "repo1") == "https://github.com/my-org/repo1" + + def test_bitbucket(self): + assert cgsr._repo_url(GitProvider.BITBUCKET_CLOUD, "my-ws", "repo1") == "https://bitbucket.org/my-ws/repo1" + + +# =========================================================================== +# (g) emit_workspace_config — BitBucket owner/provider + null p10y id +# =========================================================================== + +class TestEmitWorkspaceConfigBitbucket: + def test_bitbucket_repo_url_and_null_p10y_id(self, tmp_path): + output = str(tmp_path / "workspaces.json") + + cgsr.emit_workspace_config( + repo_id_map={"specflow-workspace1": None}, + owner="my-ws", + prefix="specflow-workspace", + workspace_pool="default", + output_path=output, + provider=GitProvider.BITBUCKET_CLOUD, + ) + + data = json.load(open(output, encoding="utf-8")) + assert data[0]["repo_url"] == "https://bitbucket.org/my-ws/specflow-workspace1" + assert data[0]["p10y_repository_id"] is None + + +# =========================================================================== +# (h) BitbucketCloudAPIClient — create/exists against mocked httpx +# =========================================================================== + +class TestBitbucketCloudAPIClient: + @pytest.mark.asyncio + async def test_repository_exists_true(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + with patch.object(client.client, "get", new=AsyncMock(return_value=MagicMock(status_code=200))): + assert await client.repository_exists("repo1") is True + await client.close() + + @pytest.mark.asyncio + async def test_repository_exists_false(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + with patch.object(client.client, "get", new=AsyncMock(return_value=MagicMock(status_code=404))): + assert await client.repository_exists("repo1") is False + await client.close() + + @pytest.mark.asyncio + async def test_create_repository_success(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + response = MagicMock(status_code=201) + response.raise_for_status = MagicMock() + response.json.return_value = {"name": "repo1"} + with patch.object(client.client, "post", new=AsyncMock(return_value=response)): + result = await client.create_repository("repo1") + assert result == {"name": "repo1"} + await client.close() + + @pytest.mark.asyncio + async def test_create_repository_already_exists_is_idempotent(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + create_response = MagicMock(status_code=400, text="Repository already exists") + get_response = MagicMock(status_code=200) + get_response.raise_for_status = MagicMock() + get_response.json.return_value = {"name": "repo1", "already_existed": True} + with ( + patch.object(client.client, "post", new=AsyncMock(return_value=create_response)), + patch.object(client.client, "get", new=AsyncMock(return_value=get_response)), + ): + result = await client.create_repository("repo1") + assert result["already_existed"] is True + await client.close() + + @pytest.mark.asyncio + async def test_create_repository_other_error_raises(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + response = MagicMock(status_code=403, text="Forbidden") + response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError("403", request=MagicMock(), response=response) + ) + with patch.object(client.client, "post", new=AsyncMock(return_value=response)): + with pytest.raises(httpx.HTTPStatusError): + await client.create_repository("repo1") + await client.close() diff --git a/backend/test/services/test_git_provider.py b/backend/test/services/test_git_provider.py new file mode 100644 index 0000000..dd322a4 --- /dev/null +++ b/backend/test/services/test_git_provider.py @@ -0,0 +1,114 @@ +"""Tests for the git-provider abstraction: resolution, sanitization, strategies.""" + +import pytest + +from app.core.config import Settings +from app.services.git_provider import ( + GitProvider, + GitProviderResolutionError, + all_strategies, + resolve_active_git_provider, + resolve_active_git_provider_from_flags, + strategy_for, +) + + +class TestResolveActiveGitProviderFromSettings: + def test_github_only_settings_resolves_github(self): + settings = Settings(GITHUB_TOKEN="gh-token", BITBUCKET_TOKEN=None, GIT_PROVIDER=None) + assert resolve_active_git_provider(settings) == GitProvider.GITHUB + + def test_bitbucket_only_settings_resolves_bitbucket(self): + settings = Settings(GITHUB_TOKEN=None, BITBUCKET_TOKEN="bb-token", GIT_PROVIDER=None) + assert resolve_active_git_provider(settings) == GitProvider.BITBUCKET_CLOUD + + def test_github_only_deployment_is_byte_identical_default(self): + """A GitHub-only deployment must resolve exactly as it did before BitBucket existed.""" + settings = Settings(GITHUB_TOKEN="gh-token", BITBUCKET_TOKEN=None, GIT_PROVIDER=None) + assert resolve_active_git_provider(settings) == GitProvider.GITHUB + + +class TestResolveActiveGitProvider: + def test_only_github_token_resolves_github(self): + provider = resolve_active_git_provider_from_flags( + override=None, has_github_token=True, has_bitbucket_token=False + ) + assert provider == GitProvider.GITHUB + + def test_only_bitbucket_token_resolves_bitbucket(self): + provider = resolve_active_git_provider_from_flags( + override=None, has_github_token=False, has_bitbucket_token=True + ) + assert provider == GitProvider.BITBUCKET_CLOUD + + def test_both_tokens_without_override_raises(self): + with pytest.raises(GitProviderResolutionError, match="Both"): + resolve_active_git_provider_from_flags( + override=None, has_github_token=True, has_bitbucket_token=True + ) + + def test_neither_token_raises(self): + with pytest.raises(GitProviderResolutionError, match="No git provider"): + resolve_active_git_provider_from_flags( + override=None, has_github_token=False, has_bitbucket_token=False + ) + + def test_explicit_override_wins_even_with_both_tokens(self): + provider = resolve_active_git_provider_from_flags( + override="bitbucket_cloud", has_github_token=True, has_bitbucket_token=True + ) + assert provider == GitProvider.BITBUCKET_CLOUD + + def test_explicit_override_is_case_insensitive(self): + provider = resolve_active_git_provider_from_flags( + override="GITHUB", has_github_token=False, has_bitbucket_token=True + ) + assert provider == GitProvider.GITHUB + + def test_invalid_override_raises(self): + with pytest.raises(GitProviderResolutionError, match="not valid"): + resolve_active_git_provider_from_flags( + override="gitlab", has_github_token=True, has_bitbucket_token=False + ) + + +class TestStrategies: + def test_github_default_git_user(self): + assert strategy_for(GitProvider.GITHUB).default_git_user == "x-access-token" + + def test_bitbucket_default_git_user(self): + assert strategy_for(GitProvider.BITBUCKET_CLOUD).default_git_user == "x-token-auth" + + def test_all_strategies_covers_both_providers(self): + providers = {s.provider for s in all_strategies()} + assert providers == {GitProvider.GITHUB, GitProvider.BITBUCKET_CLOUD} + + +class TestSanitizationPatterns: + def _sanitize(self, provider: GitProvider, message: str) -> str: + sanitized = message + for pattern, replacement in strategy_for(provider).sanitization_patterns: + sanitized = pattern.sub(replacement, sanitized) + return sanitized + + def test_github_url_scrubbed(self): + msg = "clone https://x-access-token:ghp_abc123@github.com/org/repo.git failed" + out = self._sanitize(GitProvider.GITHUB, msg) + assert "ghp_abc123" not in out + assert "[REDACTED]" in out + + def test_github_pat_token_scrubbed_standalone(self): + assert "[REDACTED]" in self._sanitize(GitProvider.GITHUB, "token=github_pat_XYZ123") + + def test_bitbucket_url_scrubbed(self): + msg = "clone https://x-token-auth:ATCTT3xFfGN0abc@bitbucket.org/ws/repo.git failed" + out = self._sanitize(GitProvider.BITBUCKET_CLOUD, msg) + assert "ATCTT3xFfGN0abc" not in out + assert "[REDACTED]" in out + + def test_bitbucket_token_scrubbed_standalone(self): + assert "[REDACTED]" in self._sanitize(GitProvider.BITBUCKET_CLOUD, "token=ATCTT3xFfGN0abc_def-2") + + def test_github_patterns_do_not_touch_bitbucket_url(self): + msg = "https://x-token-auth:ATCTT3xFfGN0abc@bitbucket.org/ws/repo.git" + assert self._sanitize(GitProvider.GITHUB, msg) == msg diff --git a/backend/test/services/test_github_auth.py b/backend/test/services/test_github_auth.py index 28dec7e..19fd323 100644 --- a/backend/test/services/test_github_auth.py +++ b/backend/test/services/test_github_auth.py @@ -10,6 +10,7 @@ reset_github_platform_secrets, ) from app.core.workspace_pool_names import DEFAULT_WORKSPACE_POOL +from app.services.git_provider import GitProvider from app.database.memory import InMemoryDatabase from app.services.github_auth import ( GithubAuthResolutionError, @@ -150,6 +151,49 @@ def patched_query(collection, filters=None, **kwargs): assert len(query_calls) == 0, "Should not use raw query for key_uid lookup" +def test_default_pool_uses_bitbucket_token_and_user_when_active(db): + key = Fernet.generate_key() + init_github_platform_secrets_for_tests( + fernet_key=key, + github_token_default=None, + git_user_name_default="x-token-auth", + bitbucket_token_default="bb-platform-token", + active_provider=GitProvider.BITBUCKET_CLOUD, + ) + try: + s = get_holder() + doc = {"workspace_pool": DEFAULT_WORKSPACE_POOL} + ctx = resolve_github_auth_for_api_key_document(doc, s) + assert ctx.token == "bb-platform-token" + assert ctx.git_user_name == "x-token-auth" + finally: + reset_github_platform_secrets() + + +def test_per_key_ciphertext_defaults_git_user_from_active_provider_strategy(db): + """When active=BitBucket and no git_user_name is stored, default to x-token-auth.""" + key = Fernet.generate_key() + init_github_platform_secrets_for_tests( + fernet_key=key, + github_token_default=None, + git_user_name_default=None, + bitbucket_token_default="bb-platform-token", + active_provider=GitProvider.BITBUCKET_CLOUD, + ) + try: + s = get_holder() + ct = s.encrypt_token("per-key-bb-token") + doc = { + "workspace_pool": DEFAULT_WORKSPACE_POOL, + "github_token_ciphertext": ct, + } + ctx = resolve_github_auth_for_api_key_document(doc, s) + assert ctx.token == "per-key-bb-token" + assert ctx.git_user_name == "x-token-auth" + finally: + reset_github_platform_secrets() + + def test_github_cli_env_for_generation(db, secrets): db.set(COL_GENERATION_SESSIONS, "gen-1", { "generation_id": "gen-1", diff --git a/backend/test/services/test_workspace_pool.py b/backend/test/services/test_workspace_pool.py index 446f9db..5d18be4 100644 --- a/backend/test/services/test_workspace_pool.py +++ b/backend/test/services/test_workspace_pool.py @@ -908,3 +908,27 @@ async def test_cleanup_workspace_wrong_state_raises(self, workspace_pool, db): }) with pytest.raises(WorkspacePoolError, match="expected 'cleaning'"): await workspace_pool.cleanup_workspace("ws-01-1") + + +class TestSanitizeTokenInMessageMultiProvider: + """_sanitize_token_in_message scrubs every provider's patterns, not just GitHub's.""" + + def test_github_url_still_scrubbed(self): + msg = "fatal: https://x-access-token:ghp_abcDEF123@github.com/org/repo.git" + out = WorkspacePoolService._sanitize_token_in_message(msg) + assert "ghp_abcDEF123" not in out + assert "[REDACTED]" in out + + def test_bitbucket_url_scrubbed(self): + msg = "fatal: https://x-token-auth:ATCTT3xFfGN0abc123@bitbucket.org/ws/repo.git" + out = WorkspacePoolService._sanitize_token_in_message(msg) + assert "ATCTT3xFfGN0abc123" not in out + assert "[REDACTED]" in out + + def test_bitbucket_token_shape_scrubbed_standalone(self): + out = WorkspacePoolService._sanitize_token_in_message("token=ATCTT3xFfGN0abc_def-2") + assert "[REDACTED]" in out + + def test_explicit_token_still_takes_priority(self): + out = WorkspacePoolService._sanitize_token_in_message("leaked my-secret-value here", token="my-secret-value") + assert "my-secret-value" not in out diff --git a/docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md b/docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md new file mode 100644 index 0000000..cbe7a16 --- /dev/null +++ b/docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md @@ -0,0 +1,230 @@ +# BitBucket Cloud support — commit, read (P10Y), and auto-provision + +> Standalone, **fully working** first delivery for BitBucket. It unlocks the +> prioritized capability end-to-end: SpecFlow agents clone/commit/push to a +> BitBucket Cloud repo, P10Y reads those commits, and the repos are +> **auto-provisioned** (no manual repo creation). CI/CD (deploy/E2E), +> Server/Data-Center, and the full `git_providers/` refactor + `github_*`→`git_*` +> renames remain out of scope (see `bitbucket-support.md`). +> +> Delivered as **two phases in one shippable change** — Phase A (credentials + +> runtime), Phase B (auto-provisioning). Both land together so the result is a +> working deployment, not a half-path. + +## ⚠️ Core assumption — a single active provider, no mixing + +**A deployment is either all-GitHub or all-BitBucket. Never mixed.** All +workspaces in the pool share one git host. This is a hard assumption and it +simplifies everything: the provider is resolved **once, globally**, from which +token is configured — not per workspace, not per key. There is no coexistence +path to build or test. + +- Active provider = explicit `GIT_PROVIDER` setting if set; else inferred from + **exactly one** of `GITHUB_TOKEN` / `BITBUCKET_TOKEN` being present. +- Both set with no explicit `GIT_PROVIDER`, or neither set → **startup error** + (ambiguous / unconfigured). Fail fast, don't guess. +- A GitHub-only deployment (only `GITHUB_TOKEN` set) resolves to GitHub and is + byte-identical to today — full back-compat, zero migration. + +## The key realization (why this is small) + +The **clone/commit/push runtime and the entire P10Y commit-reading path are +already provider-neutral**. Almost nothing that moves git data is +GitHub-specific: + +- `WorkspacePool._get_authenticated_repo_url` (`workspace_pool.py:1377`) splices + `https://{git_user_name}:{token}@{rest}` into **any** https URL — no + `github.com` anywhere. With `git_user_name="x-token-auth"` and a BitBucket URL + it already emits the exact Cloud access-token form + `https://x-token-auth:{token}@bitbucket.org/{ws}/{repo}.git`. +- All commit/push/archive go through `run_git` on `origin` (`git_utils.py`) — + host-agnostic. +- `GithubAuthContext` (`github_auth.py:16`) is just `(git_user_name, token)`; + Fernet encrypt/decrypt is opaque. +- **P10Y needs zero code change.** It reads commits two ways, both neutral: + local `git log --format=%H\t%s` (`p10y_lib.py:266`), and the external Compass + API keyed by an **integer** `repository_id` (`p10y_api_client.py`). The + `GitType` enum is dead code. Compass supports BitBucket natively; connecting it + to the BitBucket repo is a **user setup step** (Compass side, outside SpecFlow) + — same as the GitHub case today. SHAs match because both read the same repo. + +So the real GitHub-specificity to address is just: credential selection, +`git_user_name`, token log-sanitization, and repo provisioning. + +## User experience (paste one token) + +In the TUI / `.env`, below `GITHUB_TOKEN`, add `BITBUCKET_TOKEN` and +`BITBUCKET_WORKSPACE`. A BitBucket deployment sets those (and leaves +`GITHUB_TOKEN`/`GITHUB_ORG` unset); a GitHub deployment does the reverse. The +user pastes a **BitBucket Cloud Repository/Workspace Access Token** plus the +workspace slug — **no username** (access tokens always use the fixed +`x-token-auth` actor). Run the provisioning script; workspaces are created on +BitBucket and generation runs. That's it. + +--- + +## Phase A — credentials + runtime + +### 1. Minimal provider abstraction (new module) +`backend/app/services/git_provider.py` — one small file (the full +`git_providers/` package split is a later refactor PR): + +```python +class GitProvider(str, Enum): + GITHUB = "github" + BITBUCKET_CLOUD = "bitbucket_cloud" + +@dataclass(frozen=True) +class GitHostStrategy: + provider: GitProvider + default_git_user: str # x-access-token / x-token-auth + def sanitization_patterns(self) -> list[tuple[re.Pattern, str]]: ... + +_GITHUB = GitHostStrategy(GitProvider.GITHUB, "x-access-token", ...) # ghp_/gh[ps]_/github_pat_/@github.com +_BITBUCKET = GitHostStrategy(GitProvider.BITBUCKET_CLOUD, "x-token-auth", ...) # ATCTT…/x-token-auth:/@bitbucket.org + +def strategy_for(provider: GitProvider) -> GitHostStrategy: ... +def all_strategies() -> list[GitHostStrategy]: ... +def resolve_active_git_provider(settings: Settings) -> GitProvider: ... # the global switch (see assumption) +``` +- Move the existing GitHub regexes verbatim into `_GITHUB.sanitization_patterns()`; + add a golden test asserting GitHub scrubbing output is byte-identical to today. +- `resolve_active_git_provider` implements the exactly-one-token rule above. + +### 2. Settings — `BITBUCKET_TOKEN` + explicit override (mirrors GitHub) +`config.py` after line 197: +```python +BITBUCKET_TOKEN_DEFAULT: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("BITBUCKET_TOKEN_DEFAULT", "BITBUCKET_TOKEN"), +) +BITBUCKET_WORKSPACE: Optional[str] = None +GIT_PROVIDER: Optional[str] = None # explicit override; else inferred from token +K8S_SECRET_KEY_BITBUCKET_DEFAULT: str = "bitbucket-token-default" +``` +The `BITBUCKET_TOKEN` alias means the TUI/`.env` value flows in with no other +wiring — exactly how `GITHUB_TOKEN` → `GITHUB_TOKEN_DEFAULT` works today (199). + +### 3. Platform secrets — hold the active provider's token +`github_platform_secrets.py`: +- Add `bitbucket_token_default: Optional[str]` and store the resolved + `active_provider: GitProvider` on `GithubPlatformSecrets`. +- `_load_from_env` (116): read `settings.BITBUCKET_TOKEN_DEFAULT`; compute + `active_provider = resolve_active_git_provider(settings)` (fails fast on + ambiguous/unconfigured). +- `_build_secrets_from_map` (95) + `init_...` (132): read new K8s field + `settings.K8S_SECRET_KEY_BITBUCKET_DEFAULT`; same active-provider resolution. +- Add `active_default_token` / `active_git_user_default` helpers returning the + right values for the active provider. + +### 4. Auth resolution — use the active provider +`github_auth.py` `resolve_github_auth_for_api_key_document` (28): +- **git_user_name default**: when not explicitly set on the key doc, use + `strategy_for(secrets.active_provider).default_git_user` (BitBucket → + `x-token-auth`) instead of the hardcoded `"x-access-token"` (line 47). +- **default-pool token** (58): use `secrets.active_default_token` instead of the + hardcoded `github_token_default`; error message names the active provider. +- Per-key ciphertext path (42) is unchanged (opaque token); only its git_user + default becomes provider-derived. No signature change needed since the active + provider is global (read from `secrets`). + +### 5. Token sanitization — apply all providers (load-bearing safety) +`workspace_pool.py:157` `_sanitize_token_in_message`: keep the explicit `token` +replacement (line 179, host-agnostic, already covers BitBucket), then loop over +`all_strategies()` applying each provider's patterns instead of the three +hardcoded GitHub `re.sub` calls (183–199). Apply **all** providers' patterns +(cheap; defends even if a BitBucket token ever appears in a GitHub deployment's +logs). BitBucket patterns: `@bitbucket\.org` host anchor, `x-token-auth:` +prefix, `ATCTT[A-Za-z0-9_=\-]+` token shape. + +--- + +## Phase B — auto-provisioning (BitBucket Cloud REST) + +`backend/scripts/create_generation_session_repos.py`: + +- Add a `BitbucketCloudAPIClient` (parallel to the existing `GitHubAPIClient` at + `:102`), base `https://api.bitbucket.org/2.0`, header + `Authorization: Bearer {access_token}`: + - `create_repository(workspace, repo_slug)` → `POST /repositories/{workspace}/{repo_slug}` + body `{"scm": "git", "is_private": true}`; treat a 400 "already exists" as + idempotent (mirrors GitHub `repository_exists` guard). + - `repository_exists` → `GET /repositories/{workspace}/{repo_slug}` (200 = exists). + - **No team/access-grant step** — the access token's owner already has write + to repos it creates in the workspace (unlike the GitHub `--team` grant). +- Add `--git-provider {github,bitbucket_cloud}` (default: auto from configured + token via `resolve_active_git_provider`; error if ambiguous) and + `--bitbucket-workspace` (env `BITBUCKET_WORKSPACE`). +- Build `repo_url` from the provider instead of the hardcoded github.com at + 707/776: `https://bitbucket.org/{workspace}/{repo_slug}` for BitBucket. Emit it + into the workspace doc via the existing `create_workspace_document` path (629) + and the `--output-workspace-config` path. +- **P10Y metrics-enable is skipped for BitBucket** (that step hits GitHub-side + registration). `p10y_repository_id` is set externally when Compass registers + the BitBucket repo — treat BitBucket provisioning like the existing + `--skip-metrics` path and let `p10y_repository_id` come from config (or null). + +--- + +## TUI / onboarding + +`mcp_server/tui/config.py`: add `BITBUCKET_TOKEN`, `BITBUCKET_WORKSPACE` to +`ENV_SECRET_KEYS` (32) and `BITBUCKET_TOKEN` to `MASKED_KEYS` (43). Add matching +entries to `.env.quickstart.example`. The load-time parity assertion stays green. +`mcp_server/tui/onboarding.py`: present GitHub **or** BitBucket as the git step +(a provider choice, honoring the no-mixing assumption), collecting either +`GITHUB_TOKEN`/`GITHUB_ORG` or `BITBUCKET_TOKEN`/`BITBUCKET_WORKSPACE`. BitBucket +wording: "paste a BitBucket Cloud access token — no username needed." + +## P10Y — no code change +Confirmed neutral. Optionally add `BITBUCKET = "bitbucket"` to the dead `GitType` +enum for future clarity (no behavior change). Compass's BitBucket connection is a +user setup step (Compass supports it natively) — note it in the docs/PR +description as a prerequisite, just like the GitHub connection is today. + +## Persistence / back-compat (zero migration) +- No per-workspace `git_provider` field is needed — provider is global. Existing + workspace/api-key docs are read unchanged. +- All new `Settings`/secret fields default to `None`; a GitHub-only deployment + (only `GITHUB_TOKEN` set) resolves to GitHub and behaves identically. +- API-key doc unchanged (still `github_token_ciphertext`); the ciphertext is an + opaque BitBucket token in a BitBucket deployment. + +## Explicitly out of scope (later PRs) +- CI/CD (Pipelines vs Actions), deploy/E2E — `gh` CLI, `GH_TOKEN`, + `github_cli_env_for_generation`, deploy prompt recipes. Riskiest; last. +- BitBucket **Server/Data Center** (clone path `/scm/…`, arbitrary host, no + managed CI). +- Full `git_providers/` package split (hosting/provisioning/deploy ABCs) and the + `github_*`→`git_*` module/field renames + `PUT /auth/git-token` route alias. +- **Per-key / mixed-provider** support — explicitly excluded by the no-mixing + assumption. +- Notification web-URL scheme (`/tree/` vs `/src/`) — cosmetic; can ride along. + +## Tests +- Golden: GitHub sanitization + auth resolution byte-identical to current. +- `resolve_active_git_provider`: only-GitHub → GITHUB; only-BitBucket → CLOUD; + both without `GIT_PROVIDER` → error; neither → error; explicit `GIT_PROVIDER` + wins. +- Sanitizer: `ATCTT…`, `x-token-auth:@bitbucket.org` URL, explicit-token all + scrubbed; GitHub cases unchanged. +- Auth: default-pool with active=BitBucket yields git_user `x-token-auth` + the + BitBucket token; active=GitHub unchanged. +- Clone flow with active=BitBucket (mock `run_git`): authenticated URL is + `https://x-token-auth:{token}@bitbucket.org/...`. +- Provisioning: `BitbucketCloudAPIClient` create/exists with mocked `httpx`; + `--git-provider bitbucket_cloud --dry-run` emits a bitbucket.org `repo_url`; + idempotent create on 400-already-exists. +- `mcp_server` config parity assertion green with the new keys. + +Run `make unit-tests` (record baseline first; pass count ≥ baseline). Do **not** +use `make integration-tests`. `make check-complexity-diff` after (touches +`workspace_pool.py` and the provisioning script). + +## End-to-end sanity +Set `BITBUCKET_TOKEN` + `BITBUCKET_WORKSPACE` (leave GitHub vars unset), run +`create_generation_session_repos.py --git-provider bitbucket_cloud`, confirm +repos created on bitbucket.org and workspace docs carry bitbucket URLs. Run a +generation: clone → agents commit → `git push origin main` succeeds; P10Y's +local `git log` allowlist picks up the SHAs. Verify a GitHub-only deployment is +byte-identical (golden tests green). diff --git a/mcp_server/tests/test_tui_app.py b/mcp_server/tests/test_tui_app.py index 6939642..610c3b1 100644 --- a/mcp_server/tests/test_tui_app.py +++ b/mcp_server/tests/test_tui_app.py @@ -378,9 +378,9 @@ async def test_writes_chosen_provider_key_and_runs_init(self, tmp_path): assert screen.current_step_id == "provider" await self._set(screen, "OPENROUTER_API_KEY", "or-key") - await pilot.press("ctrl+n") # -> github + await pilot.press("ctrl+n") # -> git (default choice: GitHub) await pilot.pause() - assert screen.current_step_id == "github" + assert screen.current_step_id == "git" await self._set(screen, "GITHUB_TOKEN", "tok") await pilot.press("ctrl+n") # -> compass @@ -414,11 +414,11 @@ async def test_anthropic_path_writes_only_anthropic(self, tmp_path): await pilot.press("ctrl+n") # -> provider await pilot.pause() # Select the Anthropic radio (second option) and fill its key. - buttons = list(screen.query("#onboard-provider RadioButton").results()) + buttons = list(screen.query("#onboard-choice RadioButton").results()) buttons[1].value = True # toggles selection, fires RadioSet.Changed await pilot.pause() await self._set(screen, "ANTHROPIC_API_KEY", "ant-key") - await pilot.press("ctrl+n") # -> github + await pilot.press("ctrl+n") # -> git (default choice: GitHub) await pilot.pause() await self._set(screen, "GITHUB_TOKEN", "tok") await pilot.press("ctrl+n") # -> compass @@ -433,6 +433,40 @@ async def test_anthropic_path_writes_only_anthropic(self, tmp_path): assert "ANTHROPIC_API_KEY=ant-key" in env assert "OPENROUTER_API_KEY=" not in env + @pytest.mark.asyncio + async def test_bitbucket_path_writes_only_bitbucket_fields(self, tmp_path): + run_init = AsyncMock(return_value=0) + a, b, c, d, e, f = self._gate_patches(tmp_path, run_init) + with a, b, c, d, e, f: + app = tui_app.SpecFlowTUI(root=tmp_path, generation_id=None, poll_interval=999) + async with app.run_test() as pilot: + await pilot.pause() + screen = app.screen + await pilot.press("ctrl+n") # -> provider + await pilot.pause() + await self._set(screen, "OPENROUTER_API_KEY", "or-key") + await pilot.press("ctrl+n") # -> git + await pilot.pause() + assert screen.current_step_id == "git" + # Select the BitBucket radio (second option) and fill its fields. + buttons = list(screen.query("#onboard-choice RadioButton").results()) + buttons[1].value = True # toggles selection, fires RadioSet.Changed + await pilot.pause() + await self._set(screen, "BITBUCKET_TOKEN", "bb-tok") + await self._set(screen, "BITBUCKET_WORKSPACE", "my-ws") + await pilot.press("ctrl+n") # -> compass + await pilot.pause() + await self._set(screen, "P10Y_API_KEY", "p10y") + await pilot.press("ctrl+n") # -> review + await pilot.pause() + await pilot.press("ctrl+s") + await pilot.pause() + run_init.assert_awaited_once() + env = (tmp_path / ".env").read_text() + assert "BITBUCKET_TOKEN=bb-tok" in env + assert "BITBUCKET_WORKSPACE=my-ws" in env + assert "GITHUB_TOKEN=" not in env + @pytest.mark.asyncio async def test_validation_blocks_advance_without_required(self, tmp_path): run_init = AsyncMock(return_value=0) @@ -466,12 +500,12 @@ async def test_back_preserves_entered_value(self, tmp_path): await pilot.press("ctrl+n") # -> provider await pilot.pause() await self._set(screen, "OPENROUTER_API_KEY", "or-key") - await pilot.press("ctrl+n") # -> github + await pilot.press("ctrl+n") # -> git await pilot.pause() await self._set(screen, "GITHUB_ORG", "my-org") await pilot.press("ctrl+b") # back -> provider await pilot.pause() - await pilot.press("ctrl+n") # forward -> github + await pilot.press("ctrl+n") # forward -> git await pilot.pause() assert screen.query_one("#onb-GITHUB_ORG", Input).value == "my-org" @@ -484,11 +518,11 @@ async def test_secret_field_is_masked(self, tmp_path): async with app.run_test() as pilot: await pilot.pause() screen = app.screen - # Advance to the GitHub step which carries a masked + a plain field. + # Advance to the git step (default GitHub choice) which carries a masked + a plain field. await pilot.press("ctrl+n") # provider await pilot.pause() await self._set(screen, "OPENROUTER_API_KEY", "or-key") - await pilot.press("ctrl+n") # github + await pilot.press("ctrl+n") # git await pilot.pause() assert screen.query_one("#onb-GITHUB_TOKEN", Input).password is True assert screen.query_one("#onb-GIT_USER_NAME", Input).password is False diff --git a/mcp_server/tests/test_tui_onboarding.py b/mcp_server/tests/test_tui_onboarding.py index 4a2a152..fda5ae9 100644 --- a/mcp_server/tests/test_tui_onboarding.py +++ b/mcp_server/tests/test_tui_onboarding.py @@ -6,6 +6,8 @@ from tui import config, onboarding from tui.onboarding import ( + GIT_PROVIDER_BITBUCKET, + GIT_PROVIDER_GITHUB, PROVIDER_ANTHROPIC, PROVIDER_OPENROUTER, StepKind, @@ -16,16 +18,21 @@ def _all_fields(): fields = [] for step in onboarding.STEPS: fields.extend(step.fields) - fields.extend(c.field for c in step.choices) + for choice in step.choices: + fields.extend(choice.fields) return fields +def _chosen(provider=PROVIDER_OPENROUTER, git=GIT_PROVIDER_GITHUB): + return {"provider": provider, "git": git} + + class TestStepContent: def test_step_order(self): assert [s.step_id for s in onboarding.STEPS] == [ "welcome", "provider", - "github", + "git", "compass", "review", ] @@ -42,86 +49,158 @@ def test_provider_step_defaults_to_openrouter(self): assert onboarding.PROVIDER_STEP.default_choice == PROVIDER_OPENROUTER assert onboarding.PROVIDER_STEP.kind is StepKind.CHOICE - def test_optional_identity_fields_not_required(self): - github = next(s for s in onboarding.STEPS if s.step_id == "github") - by_key = {f.key: f for f in github.fields} + def test_git_step_defaults_to_github(self): + assert onboarding.GIT_STEP.default_choice == GIT_PROVIDER_GITHUB + assert onboarding.GIT_STEP.kind is StepKind.CHOICE + + def test_choice_steps_are_provider_and_git(self): + assert [s.step_id for s in onboarding.CHOICE_STEPS] == ["provider", "git"] + + def test_default_choices(self): + assert onboarding.default_choices() == { + "provider": PROVIDER_OPENROUTER, + "git": GIT_PROVIDER_GITHUB, + } + + def test_github_optional_identity_fields_not_required(self): + fields = onboarding.choice_fields(onboarding.GIT_STEP, GIT_PROVIDER_GITHUB) + by_key = {f.key: f for f in fields} assert by_key["GITHUB_TOKEN"].required is True assert by_key["GITHUB_ORG"].required is False assert by_key["GIT_USER_NAME"].required is False + def test_bitbucket_fields_both_required(self): + fields = onboarding.choice_fields(onboarding.GIT_STEP, GIT_PROVIDER_BITBUCKET) + by_key = {f.key: f for f in fields} + assert by_key["BITBUCKET_TOKEN"].required is True + assert by_key["BITBUCKET_WORKSPACE"].required is True + class TestRequiredKeys: - def test_openrouter_required_set(self): - assert set(onboarding.required_keys(PROVIDER_OPENROUTER)) == { + def test_openrouter_github_required_set(self): + assert set(onboarding.required_keys(_chosen(PROVIDER_OPENROUTER, GIT_PROVIDER_GITHUB))) == { "OPENROUTER_API_KEY", "GITHUB_TOKEN", "P10Y_API_KEY", } def test_anthropic_required_set(self): - assert set(onboarding.required_keys(PROVIDER_ANTHROPIC)) == { + assert set(onboarding.required_keys(_chosen(PROVIDER_ANTHROPIC, GIT_PROVIDER_GITHUB))) == { "ANTHROPIC_API_KEY", "GITHUB_TOKEN", "P10Y_API_KEY", } + def test_bitbucket_required_set(self): + assert set(onboarding.required_keys(_chosen(PROVIDER_OPENROUTER, GIT_PROVIDER_BITBUCKET))) == { + "OPENROUTER_API_KEY", + "BITBUCKET_TOKEN", + "BITBUCKET_WORKSPACE", + "P10Y_API_KEY", + } + class TestValidateStep: def _step(self, step_id): return next(s for s in onboarding.STEPS if s.step_id == step_id) def test_info_and_review_always_pass(self): - assert onboarding.validate_step(self._step("welcome"), {}, PROVIDER_OPENROUTER) is None - assert onboarding.validate_step(self._step("review"), {}, PROVIDER_OPENROUTER) is None + assert onboarding.validate_step(self._step("welcome"), {}, _chosen()) is None + assert onboarding.validate_step(self._step("review"), {}, _chosen()) is None def test_provider_requires_chosen_key(self): step = self._step("provider") - assert onboarding.validate_step(step, {}, PROVIDER_OPENROUTER) is not None + assert onboarding.validate_step(step, {}, _chosen(PROVIDER_OPENROUTER)) is not None ok = {"OPENROUTER_API_KEY": "k"} - assert onboarding.validate_step(step, ok, PROVIDER_OPENROUTER) is None + assert onboarding.validate_step(step, ok, _chosen(PROVIDER_OPENROUTER)) is None # The unchosen key does not satisfy the chosen provider. - assert onboarding.validate_step(step, ok, PROVIDER_ANTHROPIC) is not None + assert onboarding.validate_step(step, ok, _chosen(PROVIDER_ANTHROPIC)) is not None - def test_github_requires_token_but_not_optionals(self): - step = self._step("github") - assert onboarding.validate_step(step, {}, PROVIDER_OPENROUTER) is not None + def test_git_github_requires_token_but_not_optionals(self): + step = self._step("git") + assert onboarding.validate_step(step, {}, _chosen(git=GIT_PROVIDER_GITHUB)) is not None + assert ( + onboarding.validate_step(step, {"GITHUB_TOKEN": "t"}, _chosen(git=GIT_PROVIDER_GITHUB)) + is None + ) + + def test_git_bitbucket_requires_both_fields(self): + step = self._step("git") + chosen = _chosen(git=GIT_PROVIDER_BITBUCKET) + assert onboarding.validate_step(step, {}, chosen) is not None + assert ( + onboarding.validate_step(step, {"BITBUCKET_TOKEN": "t"}, chosen) is not None + ) assert ( - onboarding.validate_step(step, {"GITHUB_TOKEN": "t"}, PROVIDER_OPENROUTER) is None + onboarding.validate_step( + step, {"BITBUCKET_TOKEN": "t", "BITBUCKET_WORKSPACE": "ws"}, chosen + ) + is None ) + # GitHub fields left over from a prior choice don't satisfy BitBucket. + assert onboarding.validate_step(step, {"GITHUB_TOKEN": "t"}, chosen) is not None class TestValidateAll: - def _complete(self): + def _complete_github(self): return { "OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1", } - def test_complete_passes(self): - assert onboarding.validate_all(self._complete(), PROVIDER_OPENROUTER) is None + def _complete_bitbucket(self): + return { + "OPENROUTER_API_KEY": "or", + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } + + def test_complete_github_passes(self): + assert onboarding.validate_all(self._complete_github(), _chosen()) is None + + def test_complete_bitbucket_passes(self): + assert onboarding.validate_all(self._complete_bitbucket(), _chosen(git=GIT_PROVIDER_BITBUCKET)) is None def test_missing_p10y_fails(self): - vals = self._complete() + vals = self._complete_github() del vals["P10Y_API_KEY"] - assert onboarding.validate_all(vals, PROVIDER_OPENROUTER) is not None + assert onboarding.validate_all(vals, _chosen()) is not None - def test_wrong_provider_key_fails(self): + def test_wrong_llm_provider_key_fails(self): # OpenRouter set but Anthropic chosen → the chosen key is missing. - assert onboarding.validate_all(self._complete(), PROVIDER_ANTHROPIC) is not None + assert onboarding.validate_all(self._complete_github(), _chosen(PROVIDER_ANTHROPIC)) is not None + + def test_wrong_git_provider_fields_fail(self): + # GitHub fields set but BitBucket chosen → the chosen fields are missing. + assert ( + onboarding.validate_all(self._complete_github(), _chosen(git=GIT_PROVIDER_BITBUCKET)) + is not None + ) class TestEnvSatisfiesRequirements: - def test_complete_openrouter_env_passes(self): + def test_complete_openrouter_github_env_passes(self): assert onboarding.env_satisfies_requirements( {"OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1"} ) - def test_complete_anthropic_env_passes(self): + def test_complete_anthropic_github_env_passes(self): assert onboarding.env_satisfies_requirements( {"ANTHROPIC_API_KEY": "an", "GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1"} ) + def test_complete_bitbucket_env_passes(self): + assert onboarding.env_satisfies_requirements( + { + "OPENROUTER_API_KEY": "or", + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } + ) + def test_missing_llm_key_fails(self): assert not onboarding.env_satisfies_requirements( {"GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1"} @@ -132,6 +211,11 @@ def test_missing_shared_key_fails(self): {"OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh"} ) + def test_bitbucket_missing_workspace_fails(self): + assert not onboarding.env_satisfies_requirements( + {"OPENROUTER_API_KEY": "or", "BITBUCKET_TOKEN": "bb", "P10Y_API_KEY": "p1"} + ) + def test_empty_env_fails(self): assert not onboarding.env_satisfies_requirements({}) @@ -145,7 +229,7 @@ def test_drops_empty_and_unchosen_provider_key(self): "GITHUB_ORG": "", # empty → dropped "P10Y_API_KEY": "p1", } - out = onboarding.collected_secrets(values, PROVIDER_OPENROUTER) + out = onboarding.collected_secrets(values, _chosen(PROVIDER_OPENROUTER)) assert out == { "OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh", @@ -159,6 +243,22 @@ def test_keeps_present_optionals(self): "GITHUB_ORG": "my-org", "P10Y_API_KEY": "p1", } - out = onboarding.collected_secrets(values, PROVIDER_ANTHROPIC) + out = onboarding.collected_secrets(values, _chosen(PROVIDER_ANTHROPIC)) assert out["GITHUB_ORG"] == "my-org" assert "OPENROUTER_API_KEY" not in out + + def test_bitbucket_choice_drops_github_fields(self): + values = { + "OPENROUTER_API_KEY": "or", + "GITHUB_TOKEN": "gh", # left over from a prior choice → dropped + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } + out = onboarding.collected_secrets(values, _chosen(git=GIT_PROVIDER_BITBUCKET)) + assert out == { + "OPENROUTER_API_KEY": "or", + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } diff --git a/mcp_server/tui/app.py b/mcp_server/tui/app.py index 64244ef..62bbcec 100644 --- a/mcp_server/tui/app.py +++ b/mcp_server/tui/app.py @@ -1179,7 +1179,7 @@ class OnboardingScreen(_SpecFlowScreen): def __init__(self) -> None: super().__init__() self._index = 0 - self._chosen_provider = onboarding.PROVIDER_STEP.default_choice + self._chosen: dict[str, str] = onboarding.default_choices() self._values: dict[str, str] = {} @property @@ -1259,24 +1259,31 @@ def _field_rows(self, fields: tuple[onboarding.Field, ...]) -> list[Any]: return rows def _choice_widgets(self, step: onboarding.Step) -> list[Any]: + chosen_option = self._chosen[step.step_id] radio = RadioSet( *( - RadioButton(c.label, value=(c.option_id == self._chosen_provider)) + RadioButton(c.label, value=(c.option_id == chosen_option)) for c in step.choices ), - id="onboard-provider", + id="onboard-choice", ) - # The selected provider's how-to + key field are (re)built into the + # The selected option's how-to + field(s) are (re)built into the # detail container by _render_choice_detail on mount and on selection. return [radio, VerticalScroll(id="onboard-choice-detail")] def _review_widget(self) -> Any: lines: list[str] = [] - chosen = onboarding.provider_field(self._chosen_provider) - secrets = onboarding.collected_secrets(self._values, self._chosen_provider) - field_by_key = {f.key: f for s in onboarding.STEPS for f in s.fields} - field_by_key[chosen.key] = chosen - for key in [chosen.key, "GITHUB_TOKEN", "GITHUB_ORG", "GIT_USER_NAME", "P10Y_API_KEY"]: + secrets = onboarding.collected_secrets(self._values, self._chosen) + field_by_key: dict[str, onboarding.Field] = {f.key: f for s in onboarding.STEPS for f in s.fields} + review_keys: list[str] = [] + for step in onboarding.STEPS: + if step.kind is onboarding.StepKind.CHOICE: + fields = onboarding.choice_fields(step, self._chosen[step.step_id]) + field_by_key.update({f.key: f for f in fields}) + review_keys.extend(f.key for f in fields) + else: + review_keys.extend(f.key for f in step.fields) + for key in review_keys: f = field_by_key[key] if f.masked: shown = "•••• set" if secrets.get(key) else "— (missing)" @@ -1319,7 +1326,7 @@ def action_back(self) -> None: def action_next(self) -> None: self._capture() - error = onboarding.validate_step(self.current_step, self._values, self._chosen_provider) + error = onboarding.validate_step(self.current_step, self._values, self._chosen) if error: self.notify(error, severity="error") return @@ -1342,7 +1349,8 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self._save_and_init() def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - self._chosen_provider = onboarding.PROVIDER_STEP.choices[event.index].option_id + step = self.current_step + self._chosen[step.step_id] = step.choices[event.index].option_id self.call_later(self._render_choice_detail) async def _render_choice_detail(self) -> None: @@ -1351,23 +1359,22 @@ async def _render_choice_detail(self) -> None: except NoMatches: return await detail.remove_children() - choice = next( - c for c in onboarding.PROVIDER_STEP.choices if c.option_id == self._chosen_provider - ) + step = self.current_step + choice = next(c for c in step.choices if c.option_id == self._chosen[step.step_id]) widgets: list[Any] = [Static(choice.why, classes="onboard-why")] widgets.extend(self._howto_widgets(choice.how_to)) - widgets.extend(self._field_rows((choice.field,))) + widgets.extend(self._field_rows(choice.fields)) await detail.mount(*widgets) # -- save + init (unchanged behaviour) --------------------------------- def _save_and_init(self) -> None: self._capture() - error = onboarding.validate_all(self._values, self._chosen_provider) + error = onboarding.validate_all(self._values, self._chosen) if error: self.notify(error, severity="error") return - non_empty = onboarding.collected_secrets(self._values, self._chosen_provider) + non_empty = onboarding.collected_secrets(self._values, self._chosen) save_env_secrets(self.app.root, non_empty) self.query_one("#onboard-go", Button).disabled = True self.run_worker(self._run_init(), exclusive=True) diff --git a/mcp_server/tui/config.py b/mcp_server/tui/config.py index 3881026..e9fbe15 100644 --- a/mcp_server/tui/config.py +++ b/mcp_server/tui/config.py @@ -33,6 +33,8 @@ "GITHUB_TOKEN", "GIT_USER_NAME", "GITHUB_ORG", + "BITBUCKET_TOKEN", + "BITBUCKET_WORKSPACE", "P10Y_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", @@ -43,6 +45,7 @@ MASKED_KEYS: frozenset[str] = frozenset( { "GITHUB_TOKEN", + "BITBUCKET_TOKEN", "P10Y_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", diff --git a/mcp_server/tui/onboarding.py b/mcp_server/tui/onboarding.py index 93cbb8a..12d6c68 100644 --- a/mcp_server/tui/onboarding.py +++ b/mcp_server/tui/onboarding.py @@ -10,10 +10,16 @@ (``ENV_SECRET_KEYS`` / ``MASKED_KEYS``) — never restated — and a module-load assertion fails loudly if a step ever references a key that isn't a known secret, so this content cannot silently drift from the writer in ``config.py``. + +A wizard can have more than one ``CHOICE`` step (LLM provider, git host, ...). +Each is independent: the "chosen option per step" state is a +``dict[step_id, option_id]`` (see ``default_choices``), and every function +below that used to take a single "chosen provider" string now takes that dict. """ from __future__ import annotations +import itertools from dataclasses import dataclass from enum import Enum @@ -23,18 +29,17 @@ PROVIDER_OPENROUTER = "openrouter" PROVIDER_ANTHROPIC = "anthropic" -# Secret keys for the two LLM providers (exactly one is collected per run). -PROVIDER_KEYS: dict[str, str] = { - PROVIDER_OPENROUTER: "OPENROUTER_API_KEY", - PROVIDER_ANTHROPIC: "ANTHROPIC_API_KEY", -} +# Provider option ids for the git-host choice step. A deployment is either +# all-GitHub or all-BitBucket, never mixed — mirrors the backend's GitProvider. +GIT_PROVIDER_GITHUB = "github" +GIT_PROVIDER_BITBUCKET = "bitbucket_cloud" class StepKind(Enum): """What a wizard step renders/collects.""" INFO = "info" # explanatory only — no fields - CHOICE = "choice" # provider pick — reveals one field for the choice + CHOICE = "choice" # pick one option — reveals that option's field(s) FIELDS = "fields" # one or more plain input rows REVIEW = "review" # read-only recap + Save & Initialize @@ -56,14 +61,14 @@ def masked(self) -> bool: @dataclass(frozen=True) class Choice: - """One option in a ``CHOICE`` step, revealing a single field when picked.""" + """One option in a ``CHOICE`` step, revealing its field(s) when picked.""" option_id: str label: str why: str how_to: tuple[str, ...] url: str - field: Field + fields: tuple[Field, ...] @dataclass(frozen=True) @@ -123,7 +128,7 @@ class Step: "2. Create a key and paste it below.", ), url="https://openrouter.ai/keys", - field=Field("OPENROUTER_API_KEY", "OpenRouter API key", required=True), + fields=(Field("OPENROUTER_API_KEY", "OpenRouter API key", required=True),), ), Choice( option_id=PROVIDER_ANTHROPIC, @@ -134,41 +139,76 @@ class Step: "2. Create a key and paste it below.", ), url="https://console.anthropic.com/settings/keys", - field=Field("ANTHROPIC_API_KEY", "Anthropic API key", required=True), + fields=(Field("ANTHROPIC_API_KEY", "Anthropic API key", required=True),), ), ), ) -_GITHUB = Step( - step_id="github", - title="GitHub access", - kind=StepKind.FIELDS, +_GIT = Step( + step_id="git", + title="Choose your git host", + kind=StepKind.CHOICE, why=( - "A GitHub Personal Access Token lets SpecFlow create and manage the " - "disposable workspace repos where agents commit generated code. Create " - "ONE token here and reuse it for the Compass integration in the next " - "step — no second GitHub key needed." + "SpecFlow needs exactly one git host to create and manage the disposable " + "workspace repos where agents commit generated code. A deployment is " + "either all-GitHub or all-BitBucket, never mixed." ), - how_to=( - "1. Open https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", - "2. Create a classic PAT. SpecFlow always uses `repo` + `read:user` (to create", - " workspace repos and resolve your GitHub login), plus `workflow` for deploy/E2E", - " runs; `admin:repo_hook` + full `user` are added so the SAME token also works", - " for the Compass GitHub integration in the next step.", - "3. Paste it below.", - ), - url="https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", - fields=( - Field("GITHUB_TOKEN", "GitHub token", required=True), - Field( - "GITHUB_ORG", - "GitHub org (optional)", - hint="GH org for workspace repos; blank = your GitHub login", + default_choice=GIT_PROVIDER_GITHUB, + choices=( + Choice( + option_id=GIT_PROVIDER_GITHUB, + label="GitHub (default)", + why=( + "A GitHub Personal Access Token lets SpecFlow create and manage the " + "disposable workspace repos. Create ONE token here and reuse it for " + "the Compass integration in the next step — no second GitHub key needed." + ), + how_to=( + "1. Open https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", + "2. Create a classic PAT. SpecFlow always uses `repo` + `read:user` (to create", + " workspace repos and resolve your GitHub login), plus `workflow` for deploy/E2E", + " runs; `admin:repo_hook` + full `user` are added so the SAME token also works", + " for the Compass GitHub integration in the next step.", + "3. Paste it below.", + ), + url="https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", + fields=( + Field("GITHUB_TOKEN", "GitHub token", required=True), + Field( + "GITHUB_ORG", + "GitHub org (optional)", + hint="GH org for workspace repos; blank = your GitHub login", + ), + Field( + "GIT_USER_NAME", + "GitHub username (optional)", + hint="auto-resolved from your GitHub login (GET /user) when blank", + ), + ), ), - Field( - "GIT_USER_NAME", - "GitHub username (optional)", - hint="auto-resolved from your GitHub login (GET /user) when blank", + Choice( + option_id=GIT_PROVIDER_BITBUCKET, + label="BitBucket Cloud", + why=( + "A BitBucket Cloud Workspace access token lets SpecFlow create and " + "manage the disposable workspace repos. No username needed — access " + "tokens always authenticate as the fixed actor `x-token-auth`." + ), + how_to=( + "1. Open your BitBucket workspace > Settings > Access tokens > Create access token.", + "2. Grant it repository read, write, and admin scopes.", + "3. Paste the token and your workspace slug below.", + ), + url="https://bitbucket.org", + fields=( + Field("BITBUCKET_TOKEN", "BitBucket access token", required=True), + Field( + "BITBUCKET_WORKSPACE", + "BitBucket workspace", + required=True, + hint="workspace slug that owns the workspace repos", + ), + ), ), ), ) @@ -184,10 +224,9 @@ class Step: ), how_to=( "1. Create a Compass account at https://compass.p10y.com (enterprise required).", - "2. Connect your GitHub org: Settings > Integrations > New Integration > GitHub;", - " add the GH username/org owning the workspace repos; paste the SAME GitHub", - " PAT you created in the previous step (it already has the scopes Compass", - " needs: admin:repo_hook, repo:*, user:*, workflow:*); enable auto-discovery; save.", + "2. Connect your git host: Settings > Integrations > New Integration;", + " add the username/org/workspace owning the workspace repos; paste the SAME", + " token you created in the previous step; enable auto-discovery; save.", "3. Generate an API token: Settings > API Tokens > Generate.", "Docs: docs/quickstart-compass.md. init auto-resolves P10Y_ORGANISATION_ID.", ), @@ -205,10 +244,12 @@ class Step: ), ) -STEPS: tuple[Step, ...] = (_WELCOME, _PROVIDER, _GITHUB, _COMPASS, _REVIEW) +STEPS: tuple[Step, ...] = (_WELCOME, _PROVIDER, _GIT, _COMPASS, _REVIEW) -# The provider-choice step, exposed so the screen need not hardcode an index. +# Choice steps, exposed so the screen need not hardcode which steps are CHOICE. PROVIDER_STEP: Step = _PROVIDER +GIT_STEP: Step = _GIT +CHOICE_STEPS: tuple[Step, ...] = tuple(s for s in STEPS if s.kind is StepKind.CHOICE) # --------------------------------------------------------------------------- @@ -221,7 +262,8 @@ def _all_field_keys() -> set[str]: keys: set[str] = set() for step in STEPS: keys.update(f.key for f in step.fields) - keys.update(c.field.key for c in step.choices) + for choice in step.choices: + keys.update(f.key for f in choice.fields) return keys @@ -233,29 +275,52 @@ def _all_field_keys() -> set[str]: # --------------------------------------------------------------------------- # Pure validation + collection +# +# ``chosen`` is a dict[step_id, option_id] — one entry per CHOICE step. Use +# ``default_choices()`` to seed it. # --------------------------------------------------------------------------- -def provider_field(chosen_provider: str) -> Field: - """The single LLM key field for the chosen provider.""" - choice = next(c for c in _PROVIDER.choices if c.option_id == chosen_provider) - return choice.field +def default_choices() -> dict[str, str]: + """The default option id for every CHOICE step, keyed by step_id.""" + return {step.step_id: step.default_choice for step in CHOICE_STEPS} + + +def choice_fields(step: Step, chosen_option: str) -> tuple[Field, ...]: + """The field(s) revealed by the chosen option of a CHOICE step.""" + choice = next(c for c in step.choices if c.option_id == chosen_option) + return choice.fields + + +def _field_by_key() -> dict[str, Field]: + field_by_key: dict[str, Field] = {f.key: f for step in STEPS for f in step.fields} + for step in CHOICE_STEPS: + for choice in step.choices: + field_by_key.update({f.key: f for f in choice.fields}) + return field_by_key -def required_keys(chosen_provider: str) -> list[str]: +def required_keys(chosen: dict[str, str]) -> list[str]: """All keys that must be non-empty for a complete setup, in step order.""" - keys: list[str] = [provider_field(chosen_provider).key] + keys: list[str] = [] for step in STEPS: - keys.extend(f.key for f in step.fields if f.required) + if step.kind is StepKind.CHOICE: + keys.extend(f.key for f in choice_fields(step, chosen[step.step_id]) if f.required) + else: + keys.extend(f.key for f in step.fields if f.required) return keys -def validate_step(step: Step, values: dict[str, str], chosen_provider: str) -> str | None: +def validate_step(step: Step, values: dict[str, str], chosen: dict[str, str]) -> str | None: """Return an error string if the step's required inputs are unmet, else None.""" if step.kind is StepKind.CHOICE: - f = provider_field(chosen_provider) - if not values.get(f.key, "").strip(): - return f"{f.label} is required — paste it to continue." + missing = [ + f.label + for f in choice_fields(step, chosen[step.step_id]) + if f.required and not values.get(f.key, "").strip() + ] + if missing: + return "Missing required field(s): " + ", ".join(missing) return None if step.kind is StepKind.FIELDS: missing = [f.label for f in step.fields if f.required and not values.get(f.key, "").strip()] @@ -264,17 +329,12 @@ def validate_step(step: Step, values: dict[str, str], chosen_provider: str) -> s return None -def validate_all(values: dict[str, str], chosen_provider: str) -> str | None: - """Final gate before save — every required key must be present. - - Reproduces the original ``_validation_error`` rule (GitHub token + P10Y key - + exactly the chosen LLM provider key) so the contract lives in one place. - """ - field_by_key = {f.key: f for step in STEPS for f in step.fields} - field_by_key.update({c.field.key: c.field for c in _PROVIDER.choices}) +def validate_all(values: dict[str, str], chosen: dict[str, str]) -> str | None: + """Final gate before save — every required key (for the chosen options) must be present.""" + field_by_key = _field_by_key() missing = [ field_by_key[key].label - for key in required_keys(chosen_provider) + for key in required_keys(chosen) if not values.get(key, "").strip() ] if missing: @@ -283,18 +343,28 @@ def validate_all(values: dict[str, str], chosen_provider: str) -> str | None: def env_satisfies_requirements(secrets: dict[str, str]) -> bool: - """True if an existing ``.env`` already has every required key for a provider. + """True if an existing ``.env`` already has every required key for some combination + of choices (e.g. some LLM provider AND some git host). - Reuses ``validate_all`` against each known provider so the "what's required" - contract stays defined in exactly one place (no second validator). A ``.env`` - is sufficient when it satisfies either provider's requirements. + Reuses ``validate_all`` against every combination so the "what's required" + contract stays defined in exactly one place (no second validator). """ - return any(validate_all(secrets, provider) is None for provider in PROVIDER_KEYS) - - -def collected_secrets(values: dict[str, str], chosen_provider: str) -> dict[str, str]: - """Non-empty values to write to ``.env``, omitting the non-chosen provider key.""" - drop = {k for k in PROVIDER_KEYS.values() if k != provider_field(chosen_provider).key} + option_lists = [[c.option_id for c in step.choices] for step in CHOICE_STEPS] + for combo in itertools.product(*option_lists): + chosen = {step.step_id: option for step, option in zip(CHOICE_STEPS, combo)} + if validate_all(secrets, chosen) is None: + return True + return False + + +def collected_secrets(values: dict[str, str], chosen: dict[str, str]) -> dict[str, str]: + """Non-empty values to write to ``.env``, omitting fields for unchosen options.""" + drop: set[str] = set() + for step in CHOICE_STEPS: + chosen_option = chosen[step.step_id] + for choice in step.choices: + if choice.option_id != chosen_option: + drop.update(f.key for f in choice.fields) return { key: value.strip() for key, value in values.items()