diff --git a/.env.quickstart.example b/.env.quickstart.example index ab79258..6fe92f6 100644 --- a/.env.quickstart.example +++ b/.env.quickstart.example @@ -2,12 +2,25 @@ # REQUIRED — fill all of these before running specflow-init.sh # ============================================================================= +# Git host — set ONE of GITHUB_TOKEN or BITBUCKET_TOKEN below; leave the other +# blank. A deployment is either all-GitHub or all-BitBucket, never mixed. +# # GITHUB_TOKEN # What: GitHub Personal Access Token used to create and manage disposable # workspace repositories where agents commit generated code. # How: Generate a classic PAT with `repo` + `workflow` scopes: # https://github.com/settings/tokens/new?scopes=repo,workflow GITHUB_TOKEN= +# +# BITBUCKET_TOKEN / BITBUCKET_WORKSPACE (alternative to GitHub) +# What: A BitBucket Cloud Repository/Workspace access token, plus the +# workspace slug that owns the disposable workspace repos. No +# username needed — access tokens always authenticate as the fixed +# actor `x-token-auth`. +# How: Create a Workspace access token with repository read/write/admin +# scopes in BitBucket: Workspace settings > Access tokens. +# BITBUCKET_TOKEN= +# BITBUCKET_WORKSPACE= # P10Y_API_KEY # What: API key for Compass by P10Y — a code-complexity scoring service. @@ -111,6 +124,13 @@ GIT_USER_EMAIL= # or organization name. GITHUB_ORG= +# GIT_PROVIDER +# What: Explicit active git host override ("github" or "bitbucket_cloud"). +# How: Leave blank — auto-inferred from whichever of GITHUB_TOKEN / +# BITBUCKET_TOKEN is set. Only set this if you ever have both +# configured at once (which otherwise fails fast as ambiguous). +# GIT_PROVIDER= + # GIT_COMMITTER_USER_EMAIL # What: Email for system-level git commits (archive, workspace reset). # This is NOT your personal commit email — it identifies automated diff --git a/backend/app/core/config.py b/backend/app/core/config.py index bf494ad..200b700 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -199,6 +199,15 @@ def _empty_str_to_none_int(cls, v: object) -> object: validation_alias=AliasChoices("GITHUB_TOKEN_DEFAULT", "GITHUB_TOKEN"), ) + # BitBucket Cloud — mutually exclusive with GitHub (see git_provider.resolve_active_git_provider). + BITBUCKET_TOKEN_DEFAULT: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("BITBUCKET_TOKEN_DEFAULT", "BITBUCKET_TOKEN"), + ) + BITBUCKET_WORKSPACE: Optional[str] = None + # Explicit active-provider override; else inferred from exactly one of the tokens above. + GIT_PROVIDER: Optional[str] = None + # For cloning and https auth GIT_USER_NAME_DEFAULT: Optional[str] = Field( default=None, @@ -221,6 +230,7 @@ def _empty_str_to_none_int(cls, v: object) -> object: K8S_SECRET_KEY_ENCRYPTION: str = "token-encryption-key" K8S_SECRET_KEY_GITHUB_DEFAULT: str = "github-token-default" K8S_SECRET_KEY_GIT_USER_DEFAULT: str = "git-user-default" + K8S_SECRET_KEY_BITBUCKET_DEFAULT: str = "bitbucket-token-default" # Database Configuration DATABASE_TYPE: DatabaseType = DatabaseType.MEMORY diff --git a/backend/app/core/github_platform_secrets.py b/backend/app/core/github_platform_secrets.py index e5073cd..a4c31a1 100644 --- a/backend/app/core/github_platform_secrets.py +++ b/backend/app/core/github_platform_secrets.py @@ -16,6 +16,12 @@ from cryptography.fernet import Fernet, InvalidToken from app.core.config import Settings +from app.services.git_provider import ( + GitProvider, + resolve_active_git_provider, + resolve_active_git_provider_from_flags, + strategy_for, +) logger = logging.getLogger(__name__) @@ -29,6 +35,8 @@ class GithubPlatformSecrets: _fernet: Fernet github_token_default: Optional[str] git_user_name_default: Optional[str] + bitbucket_token_default: Optional[str] = None + active_provider: GitProvider = GitProvider.GITHUB def encrypt_token(self, token: str) -> str: return self._fernet.encrypt(token.encode()).decode() @@ -39,6 +47,16 @@ def decrypt_token(self, ciphertext: str) -> str: except InvalidToken as e: raise ValueError("GitHub token decryption failed") from e + @property + def active_default_token(self) -> Optional[str]: + if self.active_provider == GitProvider.BITBUCKET_CLOUD: + return self.bitbucket_token_default + return self.github_token_default + + @property + def active_git_user_default(self) -> str: + return self.git_user_name_default or strategy_for(self.active_provider).default_git_user + def get_github_platform_secrets() -> GithubPlatformSecrets: if _secrets is None: @@ -60,6 +78,8 @@ def init_github_platform_secrets_for_tests( fernet_key: bytes, github_token_default: str | None = "test-default-token", git_user_name_default: str | None = "test-user", + bitbucket_token_default: str | None = None, + active_provider: GitProvider = GitProvider.GITHUB, ) -> None: """Initialize secrets for unit tests (no K8s / .env).""" global _secrets @@ -67,6 +87,8 @@ def init_github_platform_secrets_for_tests( _fernet=Fernet(fernet_key), github_token_default=github_token_default, git_user_name_default=git_user_name_default, + bitbucket_token_default=bitbucket_token_default, + active_provider=active_provider, ) @@ -97,6 +119,8 @@ def _build_secrets_from_map( encryption_key_field: str, github_default_field: str, git_user_field: str, + bitbucket_default_field: str, + git_provider_override: str | None, ) -> GithubPlatformSecrets: enc_raw = data.get(encryption_key_field) if not enc_raw: @@ -106,10 +130,20 @@ def _build_secrets_from_map( fernet = Fernet(_decode_secret_value(enc_raw).strip().encode()) gh = data.get(github_default_field) gu = data.get(git_user_field) + bb = data.get(bitbucket_default_field) + gh_value = gh.strip() if gh else None + bb_value = bb.strip() if bb else None + active_provider = resolve_active_git_provider_from_flags( + override=git_provider_override, + has_github_token=bool(gh_value), + has_bitbucket_token=bool(bb_value), + ) return GithubPlatformSecrets( _fernet=fernet, - github_token_default=gh.strip() if gh else None, + github_token_default=gh_value, git_user_name_default=gu.strip() if gu else None, + bitbucket_token_default=bb_value, + active_provider=active_provider, ) @@ -122,10 +156,14 @@ def _load_from_env(settings: Settings) -> GithubPlatformSecrets: fernet = Fernet(str(enc).strip().encode()) gh_default = settings.GITHUB_TOKEN_DEFAULT git_user = settings.GIT_USER_NAME_DEFAULT + bb_default = settings.BITBUCKET_TOKEN_DEFAULT + active_provider = resolve_active_git_provider(settings) return GithubPlatformSecrets( _fernet=fernet, github_token_default=gh_default.strip() if gh_default else None, git_user_name_default=git_user.strip() if git_user else None, + bitbucket_token_default=bb_default.strip() if bb_default else None, + active_provider=active_provider, ) @@ -149,6 +187,8 @@ def init_github_platform_secrets(settings: Settings) -> None: encryption_key_field=settings.K8S_SECRET_KEY_ENCRYPTION, github_default_field=settings.K8S_SECRET_KEY_GITHUB_DEFAULT, git_user_field=settings.K8S_SECRET_KEY_GIT_USER_DEFAULT, + bitbucket_default_field=settings.K8S_SECRET_KEY_BITBUCKET_DEFAULT, + git_provider_override=settings.GIT_PROVIDER, ) logger.info( "Loaded GitHub platform secrets from Kubernetes API (namespace=%s, name=%s)", diff --git a/backend/app/services/git_provider.py b/backend/app/services/git_provider.py new file mode 100644 index 0000000..f497d6e --- /dev/null +++ b/backend/app/services/git_provider.py @@ -0,0 +1,107 @@ +""" +Minimal git-host provider abstraction. + +A deployment is either all-GitHub or all-BitBucket, never mixed — the active +provider is resolved once, globally, from configured tokens (or an explicit +override). See docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from app.core.config import Settings + + +class GitProvider(str, Enum): + GITHUB = "github" + BITBUCKET_CLOUD = "bitbucket_cloud" + + +class GitProviderResolutionError(Exception): + """Raised when the active git provider cannot be determined unambiguously.""" + + +@dataclass(frozen=True) +class GitHostStrategy: + provider: GitProvider + default_git_user: str + sanitization_patterns: tuple[tuple[re.Pattern[str], str], ...] + + +_GITHUB = GitHostStrategy( + provider=GitProvider.GITHUB, + default_git_user="x-access-token", + sanitization_patterns=( + (re.compile(r"(https?://[^:]+:)[^@]+(@github\.com)"), r"\1[REDACTED]\2"), + (re.compile(r"gh[ps]_[A-Za-z0-9_]+"), "[REDACTED]"), + (re.compile(r"github_pat_[A-Za-z0-9_]+"), "[REDACTED]"), + ), +) + +_BITBUCKET = GitHostStrategy( + provider=GitProvider.BITBUCKET_CLOUD, + default_git_user="x-token-auth", + sanitization_patterns=( + (re.compile(r"(https?://[^:]+:)[^@]+(@bitbucket\.org)"), r"\1[REDACTED]\2"), + (re.compile(r"ATCTT[A-Za-z0-9_=\-]+"), "[REDACTED]"), + ), +) + +_STRATEGIES: dict[GitProvider, GitHostStrategy] = { + GitProvider.GITHUB: _GITHUB, + GitProvider.BITBUCKET_CLOUD: _BITBUCKET, +} + + +def strategy_for(provider: GitProvider) -> GitHostStrategy: + return _STRATEGIES[provider] + + +def all_strategies() -> list[GitHostStrategy]: + return list(_STRATEGIES.values()) + + +def resolve_active_git_provider_from_flags( + *, + override: Optional[str], + has_github_token: bool, + has_bitbucket_token: bool, +) -> GitProvider: + """Pure resolution logic shared by settings-based and Kubernetes-secret-map loading.""" + explicit = (override or "").strip().lower() + if explicit: + try: + return GitProvider(explicit) + except ValueError as e: + valid = [p.value for p in GitProvider] + raise GitProviderResolutionError( + f"GIT_PROVIDER={explicit!r} is not valid; expected one of {valid}" + ) from e + + if has_github_token and has_bitbucket_token: + raise GitProviderResolutionError( + "Both a GitHub and a BitBucket token are configured; set GIT_PROVIDER " + "explicitly to choose the active git host (mixing providers is not supported)" + ) + if has_github_token: + return GitProvider.GITHUB + if has_bitbucket_token: + return GitProvider.BITBUCKET_CLOUD + + raise GitProviderResolutionError( + "No git provider configured; set GITHUB_TOKEN or BITBUCKET_TOKEN " + "(and GIT_PROVIDER if both might be set)" + ) + + +def resolve_active_git_provider(settings: "Settings") -> GitProvider: + return resolve_active_git_provider_from_flags( + override=settings.GIT_PROVIDER, + has_github_token=bool(settings.GITHUB_TOKEN_DEFAULT), + has_bitbucket_token=bool(settings.BITBUCKET_TOKEN_DEFAULT), + ) diff --git a/backend/app/services/github_auth.py b/backend/app/services/github_auth.py index ed8beb3..ea10638 100644 --- a/backend/app/services/github_auth.py +++ b/backend/app/services/github_auth.py @@ -44,22 +44,23 @@ def resolve_github_auth_for_api_key_document( token = secrets.decrypt_token(str(ciphertext)) except ValueError as e: raise GithubAuthResolutionError("Stored GitHub token could not be decrypted") from e - user = (per_key_user or secrets.git_user_name_default or "x-access-token").strip() + user = (per_key_user or secrets.active_git_user_default).strip() return GithubAuthContext(git_user_name=user, token=token) if pool == DEFAULT_WORKSPACE_POOL: - if not secrets.github_token_default or not secrets.git_user_name_default: + if not secrets.active_default_token or not secrets.git_user_name_default: raise GithubAuthResolutionError( - "Default workspace pool requires GITHUB_TOKEN_DEFAULT and GIT_USER_NAME_DEFAULT " - "(platform secrets) when no per-key token is stored" + f"Default workspace pool requires a default token and GIT_USER_NAME_DEFAULT " + f"(platform secrets) for the active provider ({secrets.active_provider.value}) " + f"when no per-key token is stored" ) return GithubAuthContext( git_user_name=secrets.git_user_name_default, - token=secrets.github_token_default, + token=secrets.active_default_token, ) raise GithubAuthResolutionError( - f"Workspace pool {pool!r} requires a GitHub PAT: call PUT /api/v1/auth/github-token " + f"Workspace pool {pool!r} requires a git PAT: call PUT /api/v1/auth/github-token " f"with this API key before running git operations" ) diff --git a/backend/app/services/workspace_pool.py b/backend/app/services/workspace_pool.py index 51c4ad4..af32992 100644 --- a/backend/app/services/workspace_pool.py +++ b/backend/app/services/workspace_pool.py @@ -16,7 +16,6 @@ import logging import os from pathlib import Path -import re import shutil from typing import Any, Dict, List, Optional @@ -32,6 +31,7 @@ GithubAuthResolutionError, resolve_github_auth_for_generation_id, ) +from app.services.git_provider import all_strategies from app.database.interface import IDatabase from app.state import WorkspaceStateMachine from app.state.db_adapter import StateMachineDBAdapter @@ -173,31 +173,17 @@ def _sanitize_token_in_message(message: str, token: Optional[str] = None) -> str return message sanitized = message - + # If specific token provided, replace it if token: sanitized = sanitized.replace(token, "[REDACTED]") - - # Also sanitize common patterns for GitHub tokens in URLs - # Pattern: https://username:token@github.com/... - sanitized = re.sub( - r'(https?://[^:]+:)[^@]+(@github\.com)', - r'\1[REDACTED]\2', - sanitized - ) - - # Pattern: ghp_XXXX or github_pat_XXXX tokens (GitHub personal access tokens) - sanitized = re.sub( - r'gh[ps]_[A-Za-z0-9_]+', - '[REDACTED]', - sanitized - ) - sanitized = re.sub( - r'github_pat_[A-Za-z0-9_]+', - '[REDACTED]', - sanitized - ) - + + # Sanitize known token/URL patterns for every provider, not just the active one — + # cheap, and defends even if a token from another provider leaks into these logs. + for strategy in all_strategies(): + for pattern, replacement in strategy.sanitization_patterns: + sanitized = pattern.sub(replacement, sanitized) + return sanitized def __init__(self, db: IDatabase, workspace_base_path: Optional[str | Path] = None): diff --git a/backend/scripts/create_generation_session_repos.py b/backend/scripts/create_generation_session_repos.py index f227eec..c7a506d 100755 --- a/backend/scripts/create_generation_session_repos.py +++ b/backend/scripts/create_generation_session_repos.py @@ -42,7 +42,7 @@ from pathlib import Path import sys import time -from typing import Any, Dict, List, Optional +from typing import Any, Awaitable, Callable, Dict, List, Optional from dotenv import load_dotenv import httpx @@ -66,6 +66,12 @@ from app.database.factory import get_database # noqa: E402 from app.database.firestore import FirestoreDatabase # noqa: E402 from app.database.interface import IDatabase # noqa: E402 +from app.services.git_provider import ( # noqa: E402 + GitProvider, + GitProviderResolutionError, + resolve_active_git_provider_from_flags, + strategy_for, +) from app.services.p10y.p10y_api_client import P10YInternalAPIClient # noqa: E402 LIVE_REPOSITORY_STATUS = "Live" @@ -202,93 +208,197 @@ async def close(self): await self.client.aclose() -async def create_github_repositories( - github_client: GitHubAPIClient, +class BitbucketCloudAPIClient: + """Client for BitBucket Cloud REST API repository operations.""" + + def __init__(self, access_token: str, workspace: str): + """ + Args: + access_token: BitBucket Cloud Repository/Workspace access token + workspace: BitBucket workspace slug (owner of repos) + """ + self.workspace = workspace + self.base_url = "https://api.bitbucket.org/2.0" + self.headers = {"Authorization": f"Bearer {access_token}"} + self.client = httpx.AsyncClient(timeout=30.0) + + async def create_repository(self, repo_slug: str) -> Dict[str, Any]: + """Create a private repo. Treats an already-exists 400 as idempotent success.""" + url = f"{self.base_url}/repositories/{self.workspace}/{repo_slug}" + response = await self.client.post( + url, json={"scm": "git", "is_private": True}, headers=self.headers + ) + if response.status_code == 400 and "already exists" in response.text.lower(): + return await self.get_repository(repo_slug) + response.raise_for_status() + return response.json() + + async def get_repository(self, repo_slug: str) -> Dict[str, Any]: + url = f"{self.base_url}/repositories/{self.workspace}/{repo_slug}" + response = await self.client.get(url, headers=self.headers) + response.raise_for_status() + return response.json() + + async def repository_exists(self, repo_slug: str) -> bool: + url = f"{self.base_url}/repositories/{self.workspace}/{repo_slug}" + response = await self.client.get(url, headers=self.headers) + return response.status_code == 200 + + async def close(self): + """Close the HTTP client.""" + await self.client.aclose() + + +def _repo_url(provider: GitProvider, owner: str, repo_name: str) -> str: + if provider == GitProvider.BITBUCKET_CLOUD: + return f"https://bitbucket.org/{owner}/{repo_name}" + return f"https://github.com/{owner}/{repo_name}" + + +async def _create_repositories( + client: "GitHubAPIClient | BitbucketCloudAPIClient", + provider: GitProvider, + owner: str, prefix: str, start_num: int, end_num: int, - team_slug: str | None, - delay: float = 0.1, + delay: float, + on_created: Optional[Callable[[str], Awaitable[None]]] = None, ) -> List[Dict[str, Any]]: - """ - Create multiple GitHub repositories with sequential numbering under the client's org. + """Shared create-or-skip-if-exists loop for both providers. - Args: - github_client: Initialized GitHub API client (org is github_client.org) - prefix: Repository name prefix (e.g., "generation-workspace") - start_num: Starting number (inclusive) - end_num: Ending number (inclusive) - team_slug: If set, grant this team Write access (push) on each repo - delay: Delay between requests in seconds (default: 0.1) - - Returns: - List of created repository data + ``on_created`` runs after each repo (create or already-exists) — used for + GitHub's per-repo team-grant step; BitBucket has no equivalent. """ - created_repos = [] - org = github_client.org + created_repos: List[Dict[str, Any]] = [] for num in range(start_num, end_num + 1): repo_name = f"{prefix}{num}" - # Check if repo already exists - if await github_client.repository_exists(repo_name): + if await client.repository_exists(repo_name): print(f"⚠️ Repository '{repo_name}' already exists, skipping creation") - # Still add to list for P10y sync created_repos.append({ "name": repo_name, - "full_name": f"{org}/{repo_name}", - "html_url": f"https://github.com/{org}/{repo_name}", + "full_name": f"{owner}/{repo_name}", + "html_url": _repo_url(provider, owner, repo_name), "already_existed": True, }) else: try: - print(f"📦 Creating repository: {org}/{repo_name}") - repo_data = await github_client.create_repository(repo_name) + print(f"📦 Creating repository: {owner}/{repo_name}") + repo_data = await client.create_repository(repo_name) created_repos.append(repo_data) - print(f"✅ Created: {repo_data['html_url']}") + print(f"✅ Created: {_repo_url(provider, owner, repo_name)}") except httpx.HTTPStatusError as e: print(f"❌ Failed to create {repo_name}: {e}") print(f" Response: {e.response.text}") raise - if team_slug: - try: - await github_client.add_team_repository_write(team_slug, repo_name) - print(f" 👥 Team '{team_slug}' granted Write on {repo_name}") - except httpx.HTTPStatusError as e: - print(f"❌ Failed to add team {team_slug} to {repo_name}: {e}") - print(f" Response: {e.response.text}") - raise + if on_created is not None: + await on_created(repo_name) - # Add delay to avoid rate limiting if num < end_num: await asyncio.sleep(delay) return created_repos +async def create_bitbucket_repositories( + bitbucket_client: BitbucketCloudAPIClient, + prefix: str, + start_num: int, + end_num: int, + delay: float = 0.1, +) -> List[Dict[str, Any]]: + """Create multiple BitBucket Cloud repositories with sequential numbering. + + No team-grant step: the access token's owner already has write access to + repos it creates in the workspace. + """ + return await _create_repositories( + bitbucket_client, + GitProvider.BITBUCKET_CLOUD, + bitbucket_client.workspace, + prefix, + start_num, + end_num, + delay, + ) + + +async def create_github_repositories( + github_client: GitHubAPIClient, + prefix: str, + start_num: int, + end_num: int, + team_slug: str | None, + delay: float = 0.1, +) -> List[Dict[str, Any]]: + """ + Create multiple GitHub repositories with sequential numbering under the client's org. + + Args: + github_client: Initialized GitHub API client (org is github_client.org) + prefix: Repository name prefix (e.g., "generation-workspace") + start_num: Starting number (inclusive) + end_num: Ending number (inclusive) + team_slug: If set, grant this team Write access (push) on each repo + delay: Delay between requests in seconds (default: 0.1) + + Returns: + List of created repository data + """ + def _team_grant_hook(slug: str) -> Callable[[str], Awaitable[None]]: + async def _grant_team(repo_name: str) -> None: + try: + await github_client.add_team_repository_write(slug, repo_name) + print(f" 👥 Team '{slug}' granted Write on {repo_name}") + except httpx.HTTPStatusError as e: + print(f"❌ Failed to add team {slug} to {repo_name}: {e}") + print(f" Response: {e.response.text}") + raise + + return _grant_team + + return await _create_repositories( + github_client, + GitProvider.GITHUB, + github_client.org, + prefix, + start_num, + end_num, + delay, + on_created=_team_grant_hook(team_slug) if team_slug else None, + ) + + def print_dry_run_plan( *, + provider: GitProvider, token_login: str, - github_org: str, - team_slug: str | None, + owner: str, + team_slug: Optional[str], skip_team: bool, skip_github: bool, prefix: str, start_num: int, end_num: int, ) -> None: - """Print planned GitHub org, team, token actor, and full repo URLs (no API calls).""" + """Print planned provider owner, team (GitHub only), token actor, and full repo URLs (no API calls).""" + is_github = provider == GitProvider.GITHUB print("\n" + "=" * 80) - print("🔍 DRY RUN — GitHub stage only (no repos created, no team grants, no P10y/Firestore)") + print(f"🔍 DRY RUN — {provider.value} stage only (no repos created, no team grants, no P10y/Firestore)") print("=" * 80) print( " Token authenticates as: " f"{token_login}\n" - " (This identity must have permission to create repos in the org and manage team access.)" + " (This identity must have permission to create repos in the org/workspace" + + (" and manage team access." if is_github else ".") ) - print(f" Organization (repo owner): {github_org}") - if skip_github: + print(f" Organization/workspace (repo owner): {owner}") + if not is_github: + print(" Team Write (push): — (n/a for BitBucket; token owner already has write)") + elif skip_github: print(" Team Write (push): — (--skip-github; would not run GitHub API)") elif skip_team: print(" Team Write (push): — (--skip-team)") @@ -300,19 +410,24 @@ def print_dry_run_plan( print(" Repository full URLs (same as Firestore repo_url):") for num in range(start_num, end_num + 1): repo_name = f"{prefix}{num}" - print(f" https://github.com/{github_org}/{repo_name}") + print(f" {_repo_url(provider, owner, repo_name)}") print() if skip_github: - print(" Would skip: POST /orgs/{org}/repos (no new repositories)") - else: - print(f" Would create {end_num - start_num + 1} private repos: POST /orgs/{github_org}/repos") + print(" Would skip: repository creation API call (no new repositories)") + elif is_github: + print(f" Would create {end_num - start_num + 1} private repos: POST /orgs/{owner}/repos") if not skip_team and team_slug: print( f" Would grant team '{team_slug}' Write on each: " - f"PUT /orgs/{github_org}/teams/{team_slug}/repos/{github_org}/" + f"PUT /orgs/{owner}/teams/{team_slug}/repos/{owner}/" ) elif skip_team: print(" Would skip: team repository permission updates (--skip-team)") + else: + print( + f" Would create {end_num - start_num + 1} private repos: " + f"POST /repositories/{owner}/ (BitBucket Cloud)" + ) print("=" * 80) @@ -603,7 +718,7 @@ def create_workspace_document( workspace_id: str, set_number: int, repo_url: str, - p10y_repository_id: int, + p10y_repository_id: Optional[int], now: datetime, workspace_pool: str = "default" ) -> dict: @@ -651,13 +766,14 @@ def create_workspace_document( async def add_workspaces_to_firestore( - repo_id_map: Dict[str, int], - github_org: str, + repo_id_map: Dict[str, Optional[int]], + owner: str, prefix: str, start_num: int, workspace_pool: str = "default", firestore_project_id: Optional[str] = None, firestore_database_id: Optional[str] = None, + provider: GitProvider = GitProvider.GITHUB, ) -> None: """ Add workspace entries to Firestore database. @@ -704,7 +820,7 @@ async def add_workspaces_to_firestore( workspace_id = f"ws-{set_number:02d}-{workspace_index}" # Create repo URL - repo_url = f"https://github.com/{github_org}/{repo_name}" + repo_url = _repo_url(provider, owner, repo_name) # Create workspace document workspace_doc = create_workspace_document( @@ -743,26 +859,26 @@ async def add_workspaces_to_firestore( def emit_workspace_config( - repo_id_map: Dict[str, int], - github_org: str, + repo_id_map: Dict[str, Optional[int]], + owner: str, prefix: str, workspace_pool: str, output_path: str, ordered_repos: Optional[List[str]] = None, + provider: GitProvider = GitProvider.GITHUB, ) -> None: """ Write a JSON workspace-config file in the exact schema consumed by ``init_firestore.py --workspace-config``: [{"workspace_id": str, "repo_url": str, - "p10y_repository_id": int, "workspace_pool": str}, ...] - - All four fields are required (matching WorkspaceConfig dataclass). + "p10y_repository_id": int | None, "workspace_pool": str}, ...] Field sources: - workspace_id: derived the same way as add_workspaces_to_firestore (ws-{set:02d}-{idx}) - - repo_url: https://github.com/{github_org}/{repo_name} - - p10y_repository_id: integer value from repo_id_map[repo_name] + - repo_url: built from ``provider`` (github.com or bitbucket.org) + owner + repo_name + - p10y_repository_id: value from repo_id_map[repo_name]; null when P10Y lookup was + skipped (e.g. BitBucket provisioning — registered externally once Compass connects it) - workspace_pool: workspace_pool argument When ordered_repos is provided (the --repos path), workspace IDs are assigned @@ -771,10 +887,11 @@ def emit_workspace_config( def _make_entry(repo_name: str, idx: int) -> Dict[str, Any]: set_number = (idx // 3) + 1 workspace_index = (idx % 3) + 1 + p10y_id = repo_id_map[repo_name] return { "workspace_id": f"ws-{set_number:02d}-{workspace_index}", - "repo_url": f"https://github.com/{github_org}/{repo_name}", - "p10y_repository_id": int(repo_id_map[repo_name]), + "repo_url": _repo_url(provider, owner, repo_name), + "p10y_repository_id": int(p10y_id) if p10y_id is not None else None, "workspace_pool": workspace_pool, } @@ -851,6 +968,24 @@ async def main(): metavar="ORG", help="GitHub organization login; repos are ORG/{PREFIX}N. Env: GITHUB_ORG, GITHUB_ORG_DEFAULT", ) + parser.add_argument( + "--git-provider", + type=str, + choices=[p.value for p in GitProvider], + default=None, + help=( + "Active git host. Default: inferred from whichever of GITHUB_TOKEN / " + "BITBUCKET_TOKEN is configured (error if both or neither, unless GIT_PROVIDER " + "is set in the environment)." + ), + ) + parser.add_argument( + "--bitbucket-workspace", + type=str, + default=os.getenv("BITBUCKET_WORKSPACE"), + metavar="WORKSPACE", + help="BitBucket Cloud workspace slug; repos are WORKSPACE/{PREFIX}N. Env: BITBUCKET_WORKSPACE", + ) parser.add_argument( "--team", type=str, @@ -946,37 +1081,47 @@ async def main(): workspace_pool = args.workspace_pool.lower() if args.workspace_pool else "default" github_org = (args.github_org or "").strip() or None + bitbucket_workspace = (args.bitbucket_workspace or "").strip() or None team_slug = None if args.skip_team else ((args.team or "").strip() or None) gcp_cli = (args.gcp_project or "").strip() or None fsdb_cli = (args.firestore_database or "").strip() or None firestore_target_from_cli = bool(gcp_cli and fsdb_cli) + try: + provider = resolve_active_git_provider_from_flags( + override=args.git_provider or cfg.GIT_PROVIDER, + has_github_token=bool(cfg.GITHUB_TOKEN_DEFAULT), + has_bitbucket_token=bool(cfg.BITBUCKET_TOKEN_DEFAULT), + ) + except GitProviderResolutionError as e: + print(f"❌ Error: {e}") + sys.exit(1) + + is_bitbucket = provider == GitProvider.BITBUCKET_CLOUD + if is_bitbucket: + # No Compass/GitHub registration step for BitBucket in this PR (see plan doc); + # p10y_repository_id is set externally once Compass connects the repo. + args.skip_metrics = True + owner = bitbucket_workspace if is_bitbucket else github_org + owner_flag = "--bitbucket-workspace (or BITBUCKET_WORKSPACE)" if is_bitbucket else "--github-org (or GITHUB_ORG / GITHUB_ORG_DEFAULT)" + # Validate arguments if args.start > args.end: print("❌ Error: start number must be less than or equal to end number") sys.exit(1) - if args.dry_run and not github_org: - print( - "❌ Error: --dry-run requires --github-org (or GITHUB_ORG / GITHUB_ORG_DEFAULT) " - "to list repository URLs" - ) + if args.dry_run and not owner: + print(f"❌ Error: --dry-run requires {owner_flag} to list repository URLs") sys.exit(1) if not args.skip_github: - if not github_org: - print( - "❌ Error: --github-org is required to create repositories " - "(or set GITHUB_ORG / GITHUB_ORG_DEFAULT)" - ) + if not owner: + print(f"❌ Error: {owner_flag} is required to create repositories") sys.exit(1) - if not args.dry_run and not args.skip_firestore and not github_org: - print( - "❌ Error: --github-org is required for Firestore repo URLs " - "(or set GITHUB_ORG / GITHUB_ORG_DEFAULT)" - ) + if not args.dry_run and not args.skip_firestore and not owner: + print(f"❌ Error: {owner_flag} is required for Firestore repo URLs") sys.exit(1) if not args.skip_firestore and not args.dry_run: @@ -998,16 +1143,19 @@ async def main(): # Check required environment variables github_token = cfg.GITHUB_TOKEN_DEFAULT + bitbucket_token = cfg.BITBUCKET_TOKEN_DEFAULT + token = bitbucket_token if is_bitbucket else github_token + token_env_flag = "BITBUCKET_TOKEN" if is_bitbucket else "GITHUB_TOKEN_DEFAULT (or legacy GITHUB_TOKEN)" p10y_api_key = cfg.P10Y_API_KEY # gitleaks:allow - variable assignment, not a literal p10y_base_url = cfg.P10Y_BASE_URL p10y_org_id = cfg.P10Y_ORGANISATION_ID git_username = cfg.GIT_USER_NAME_DEFAULT - if not args.dry_run and not github_token: - print("❌ Error: GITHUB_TOKEN_DEFAULT (or legacy GITHUB_TOKEN) not set in environment") + if not args.dry_run and not token: + print(f"❌ Error: {token_env_flag} not set in environment") sys.exit(1) - if not args.dry_run: + if not args.dry_run and not is_bitbucket: if not p10y_api_key: print("❌ Error: P10Y_API_KEY not set in environment") sys.exit(1) @@ -1016,14 +1164,14 @@ async def main(): print("❌ Error: P10Y_ORGANISATION_ID not set in environment") sys.exit(1) - if args.dry_run and not github_token: - print( - "⚠️ GITHUB_TOKEN_DEFAULT not set — resolve token actor via " - "GIT_USER_NAME_DEFAULT or add a token for GET /user" - ) + if args.dry_run and not token: + print(f"⚠️ {token_env_flag} not set — resolve token actor via GIT_USER_NAME_DEFAULT or add a token") + if is_bitbucket: + # BitBucket access tokens always authenticate as a fixed actor — no username to resolve. + git_username = strategy_for(provider).default_git_user # Optional: resolve token owner's login for logging (org repos do not use this as owner) - if not git_username and github_token: + elif not git_username and github_token: print("⚠️ GIT_USER_NAME_DEFAULT not set, fetching token owner from GitHub API...") try: temp_client = GitHubAPIClient(github_token, "_") @@ -1056,10 +1204,12 @@ async def main(): print(f" Prefix: {args.prefix}") print(f" Range: {args.start} to {args.end}") print(f" Count: {args.end - args.start + 1} repositories") - print(f" GitHub org (repo owner): {github_org or '—'}") - print(f" Team Write (slug): {team_slug or '—'}") + print(f" Git provider: {provider.value}") + print(f" Organization/workspace (repo owner): {owner or '—'}") + if not is_bitbucket: + print(f" Team Write (slug): {team_slug or '—'}") print(f" Token login (info): {git_username}") - if not args.dry_run: + if not args.dry_run and not is_bitbucket: print(f" P10y Org ID: {p10y_org_id}") print(f" Workspace Pool: {workspace_pool}") if firestore_target_from_cli: @@ -1074,8 +1224,9 @@ async def main(): if args.dry_run: print_dry_run_plan( + provider=provider, token_login=git_username, - github_org=github_org or "", + owner=owner or "", team_slug=team_slug, skip_team=args.skip_team, skip_github=args.skip_github, @@ -1083,75 +1234,95 @@ async def main(): start_num=args.start, end_num=args.end, ) - print("\n✅ Dry run finished — exited before GitHub API calls, P10y, and Firestore.") + print("\n✅ Dry run finished — exited before any provider API calls, P10y, and Firestore.") return - github_client = GitHubAPIClient(github_token, github_org or "_") + provider_client: GitHubAPIClient | BitbucketCloudAPIClient + if is_bitbucket: + provider_client = BitbucketCloudAPIClient(token, owner or "_") + else: + provider_client = GitHubAPIClient(token, owner or "_") p10y_client = P10YInternalAPIClient(base_url=p10y_base_url, api_key=p10y_api_key) - + try: - # Step 1: Create GitHub repositories + # Step 1: Create repositories if own_repo_list is not None: # --repos path: repos already exist, skip creation entirely - print("\n⏭️ Skipping GitHub repository creation (--repos provided)") + print("\n⏭️ Skipping repository creation (--repos provided)") created_repos = [] repo_names = own_repo_list - elif not args.skip_github: - created_repos = await create_github_repositories( - github_client, - args.prefix, - args.start, - args.end, - team_slug, - args.delay, - ) - print(f"\n✅ Created/found {len(created_repos)} repositories") - repo_names = [f"{args.prefix}{num}" for num in range(args.start, args.end + 1)] else: - print("\n⏭️ Skipping GitHub repository creation") - created_repos = [] repo_names = [f"{args.prefix}{num}" for num in range(args.start, args.end + 1)] + if not args.skip_github: + if is_bitbucket: + created_repos = await create_bitbucket_repositories( + provider_client, + args.prefix, + args.start, + args.end, + args.delay, + ) + else: + created_repos = await create_github_repositories( + provider_client, + args.prefix, + args.start, + args.end, + team_slug, + args.delay, + ) + print(f"\n✅ Created/found {len(created_repos)} repositories") + else: + print("\n⏭️ Skipping repository creation") + created_repos = [] + + if is_bitbucket: + # No Compass/P10Y registration in this PR — external setup connects the repo + # later, and p10y_repository_id stays null until it does. + print("\n⏭️ Skipping P10Y repository ID lookup (BitBucket provisioning)") + repo_id_map: Dict[str, Optional[int]] = {name: None for name in repo_names} + else: + # Step 2: Get P10y repository IDs + # For --repos, search with empty prefix to match arbitrary names across the full org. + p10y_search_prefix = "" if own_repo_list is not None else args.prefix + repo_id_map = await get_repository_ids( + p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org + ) - # Step 2: Get P10y repository IDs - # For --repos, search with empty prefix to match arbitrary names across the full org. - p10y_search_prefix = "" if own_repo_list is not None else args.prefix - repo_id_map = await get_repository_ids( - p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org - ) + # Newly created GitHub repos are invisible to P10Y until Compass re-fetches the + # connection that owns them. When some are missing, trigger a re-fetch, wait, and + # look up again (twice). Failing to do this is what let an expansion run (K=1 → K=3) + # silently write a too-small workspaces.json and under-seed Firestore. + missing = [r for r in repo_names if r not in repo_id_map] + if missing: + print(f"\n🔄 {len(missing)} repo(s) not in P10Y yet: {', '.join(missing)} — triggering re-fetch ...") + await trigger_repository_refetch(p10y_client, p10y_org_id, github_org, repo_names) - # Newly created GitHub repos are invisible to P10Y until Compass re-fetches the - # connection that owns them. When some are missing, trigger a re-fetch, wait, and - # look up again (twice). Failing to do this is what let an expansion run (K=1 → K=3) - # silently write a too-small workspaces.json and under-seed Firestore. - missing = [r for r in repo_names if r not in repo_id_map] - if missing: - print(f"\n🔄 {len(missing)} repo(s) not in P10Y yet: {', '.join(missing)} — triggering re-fetch ...") - await trigger_repository_refetch(p10y_client, p10y_org_id, github_org, repo_names) - - deadline = time.time() + P10Y_REFETCH_TIMEOUT_SECONDS - while missing and time.time() < deadline: - print(f" ⏱️ {len(missing)} still missing; re-checking in {P10Y_REFETCH_POLL_SECONDS}s ...") - await asyncio.sleep(P10Y_REFETCH_POLL_SECONDS) - repo_id_map = await get_repository_ids( - p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org - ) - missing = [r for r in repo_names if r not in repo_id_map] + deadline = time.time() + P10Y_REFETCH_TIMEOUT_SECONDS + while missing and time.time() < deadline: + print(f" ⏱️ {len(missing)} still missing; re-checking in {P10Y_REFETCH_POLL_SECONDS}s ...") + await asyncio.sleep(P10Y_REFETCH_POLL_SECONDS) + repo_id_map = await get_repository_ids( + p10y_client, p10y_org_id, repo_names, p10y_search_prefix, github_org + ) + missing = [r for r in repo_names if r not in repo_id_map] - if missing: - print(f"\n❌ Could not resolve P10Y IDs for {len(missing)} repo(s): {', '.join(missing)}.\nExiting the script after {P10Y_REFETCH_TIMEOUT_SECONDS} seconds. Potential debugging: Verify if on P10Y UI repo list, try re-fetching them manually, verify Integration to Github") - sys.exit(1) + if missing: + print(f"\n❌ Could not resolve P10Y IDs for {len(missing)} repo(s): {', '.join(missing)}.\nExiting the script after {P10Y_REFETCH_TIMEOUT_SECONDS} seconds. Potential debugging: Verify if on P10Y UI repo list, try re-fetching them manually, verify Integration to Github") + sys.exit(1) - repo_ids = list(repo_id_map.values()) + repo_ids = [rid for rid in repo_id_map.values() if rid is not None] # Emit workspace config JSON if requested (schema matches init_firestore.py --workspace-config) if args.output_workspace_config: emit_workspace_config( repo_id_map=repo_id_map, - github_org=github_org or "", + owner=owner or "", prefix=args.prefix, workspace_pool=workspace_pool, output_path=args.output_workspace_config, ordered_repos=own_repo_list, + provider=provider, ) # Step 4: Start metrics calculation only for repos that are not already Live. @@ -1186,12 +1357,13 @@ async def main(): if not args.skip_firestore and own_repo_list is None: await add_workspaces_to_firestore( repo_id_map, - github_org, + owner, args.prefix, args.start, workspace_pool, firestore_project_id=gcp_cli if firestore_target_from_cli else None, firestore_database_id=fsdb_cli if firestore_target_from_cli else None, + provider=provider, ) else: print("\n⏭️ Skipping Firestore workspace creation") @@ -1216,7 +1388,7 @@ async def main(): traceback.print_exc() sys.exit(1) finally: - await github_client.close() + await provider_client.close() await p10y_client.close() print("\n✅ Script completed successfully!") diff --git a/backend/test/scripts/test_create_generation_session_repos.py b/backend/test/scripts/test_create_generation_session_repos.py index 50a0368..b3b2d27 100644 --- a/backend/test/scripts/test_create_generation_session_repos.py +++ b/backend/test/scripts/test_create_generation_session_repos.py @@ -16,11 +16,13 @@ import json import re from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import scripts.create_generation_session_repos as cgsr +from app.services.git_provider import GitProvider from scripts.init_firestore import WorkspaceConfig # --------------------------------------------------------------------------- @@ -48,7 +50,7 @@ def test_roundtrip_through_workspace_config(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="test-org", + owner="test-org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -72,7 +74,7 @@ def test_all_four_required_fields_present(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="my-org", + owner="my-org", prefix="specflow-workspace", workspace_pool="testpool", output_path=output, @@ -95,7 +97,7 @@ def test_workspace_id_derivation(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -113,7 +115,7 @@ def test_repo_url_uses_github_org(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="my-org", + owner="my-org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -129,7 +131,7 @@ def test_p10y_repository_id_is_int_not_bool(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -147,7 +149,7 @@ def test_workspace_pool_propagated(self, tmp_path): cgsr.emit_workspace_config( repo_id_map=repo_id_map, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="custom-pool", output_path=output, @@ -162,7 +164,7 @@ def test_output_file_created_in_subdir(self, tmp_path): cgsr.emit_workspace_config( repo_id_map={"specflow-workspace1": 1}, - github_org="org", + owner="org", prefix="specflow-workspace", workspace_pool="default", output_path=output, @@ -444,3 +446,95 @@ def test_script_file_exists_on_disk(self): assert _SCRIPT_FILE.exists(), ( f"Script not found at expected path: {_SCRIPT_FILE}" ) + + +# =========================================================================== +# (f) _repo_url — provider-generic URL construction +# =========================================================================== + +class TestRepoUrl: + def test_github(self): + assert cgsr._repo_url(GitProvider.GITHUB, "my-org", "repo1") == "https://github.com/my-org/repo1" + + def test_bitbucket(self): + assert cgsr._repo_url(GitProvider.BITBUCKET_CLOUD, "my-ws", "repo1") == "https://bitbucket.org/my-ws/repo1" + + +# =========================================================================== +# (g) emit_workspace_config — BitBucket owner/provider + null p10y id +# =========================================================================== + +class TestEmitWorkspaceConfigBitbucket: + def test_bitbucket_repo_url_and_null_p10y_id(self, tmp_path): + output = str(tmp_path / "workspaces.json") + + cgsr.emit_workspace_config( + repo_id_map={"specflow-workspace1": None}, + owner="my-ws", + prefix="specflow-workspace", + workspace_pool="default", + output_path=output, + provider=GitProvider.BITBUCKET_CLOUD, + ) + + data = json.load(open(output, encoding="utf-8")) + assert data[0]["repo_url"] == "https://bitbucket.org/my-ws/specflow-workspace1" + assert data[0]["p10y_repository_id"] is None + + +# =========================================================================== +# (h) BitbucketCloudAPIClient — create/exists against mocked httpx +# =========================================================================== + +class TestBitbucketCloudAPIClient: + @pytest.mark.asyncio + async def test_repository_exists_true(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + with patch.object(client.client, "get", new=AsyncMock(return_value=MagicMock(status_code=200))): + assert await client.repository_exists("repo1") is True + await client.close() + + @pytest.mark.asyncio + async def test_repository_exists_false(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + with patch.object(client.client, "get", new=AsyncMock(return_value=MagicMock(status_code=404))): + assert await client.repository_exists("repo1") is False + await client.close() + + @pytest.mark.asyncio + async def test_create_repository_success(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + response = MagicMock(status_code=201) + response.raise_for_status = MagicMock() + response.json.return_value = {"name": "repo1"} + with patch.object(client.client, "post", new=AsyncMock(return_value=response)): + result = await client.create_repository("repo1") + assert result == {"name": "repo1"} + await client.close() + + @pytest.mark.asyncio + async def test_create_repository_already_exists_is_idempotent(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + create_response = MagicMock(status_code=400, text="Repository already exists") + get_response = MagicMock(status_code=200) + get_response.raise_for_status = MagicMock() + get_response.json.return_value = {"name": "repo1", "already_existed": True} + with ( + patch.object(client.client, "post", new=AsyncMock(return_value=create_response)), + patch.object(client.client, "get", new=AsyncMock(return_value=get_response)), + ): + result = await client.create_repository("repo1") + assert result["already_existed"] is True + await client.close() + + @pytest.mark.asyncio + async def test_create_repository_other_error_raises(self): + client = cgsr.BitbucketCloudAPIClient("tok", "my-ws") + response = MagicMock(status_code=403, text="Forbidden") + response.raise_for_status = MagicMock( + side_effect=httpx.HTTPStatusError("403", request=MagicMock(), response=response) + ) + with patch.object(client.client, "post", new=AsyncMock(return_value=response)): + with pytest.raises(httpx.HTTPStatusError): + await client.create_repository("repo1") + await client.close() diff --git a/backend/test/services/test_git_provider.py b/backend/test/services/test_git_provider.py new file mode 100644 index 0000000..dd322a4 --- /dev/null +++ b/backend/test/services/test_git_provider.py @@ -0,0 +1,114 @@ +"""Tests for the git-provider abstraction: resolution, sanitization, strategies.""" + +import pytest + +from app.core.config import Settings +from app.services.git_provider import ( + GitProvider, + GitProviderResolutionError, + all_strategies, + resolve_active_git_provider, + resolve_active_git_provider_from_flags, + strategy_for, +) + + +class TestResolveActiveGitProviderFromSettings: + def test_github_only_settings_resolves_github(self): + settings = Settings(GITHUB_TOKEN="gh-token", BITBUCKET_TOKEN=None, GIT_PROVIDER=None) + assert resolve_active_git_provider(settings) == GitProvider.GITHUB + + def test_bitbucket_only_settings_resolves_bitbucket(self): + settings = Settings(GITHUB_TOKEN=None, BITBUCKET_TOKEN="bb-token", GIT_PROVIDER=None) + assert resolve_active_git_provider(settings) == GitProvider.BITBUCKET_CLOUD + + def test_github_only_deployment_is_byte_identical_default(self): + """A GitHub-only deployment must resolve exactly as it did before BitBucket existed.""" + settings = Settings(GITHUB_TOKEN="gh-token", BITBUCKET_TOKEN=None, GIT_PROVIDER=None) + assert resolve_active_git_provider(settings) == GitProvider.GITHUB + + +class TestResolveActiveGitProvider: + def test_only_github_token_resolves_github(self): + provider = resolve_active_git_provider_from_flags( + override=None, has_github_token=True, has_bitbucket_token=False + ) + assert provider == GitProvider.GITHUB + + def test_only_bitbucket_token_resolves_bitbucket(self): + provider = resolve_active_git_provider_from_flags( + override=None, has_github_token=False, has_bitbucket_token=True + ) + assert provider == GitProvider.BITBUCKET_CLOUD + + def test_both_tokens_without_override_raises(self): + with pytest.raises(GitProviderResolutionError, match="Both"): + resolve_active_git_provider_from_flags( + override=None, has_github_token=True, has_bitbucket_token=True + ) + + def test_neither_token_raises(self): + with pytest.raises(GitProviderResolutionError, match="No git provider"): + resolve_active_git_provider_from_flags( + override=None, has_github_token=False, has_bitbucket_token=False + ) + + def test_explicit_override_wins_even_with_both_tokens(self): + provider = resolve_active_git_provider_from_flags( + override="bitbucket_cloud", has_github_token=True, has_bitbucket_token=True + ) + assert provider == GitProvider.BITBUCKET_CLOUD + + def test_explicit_override_is_case_insensitive(self): + provider = resolve_active_git_provider_from_flags( + override="GITHUB", has_github_token=False, has_bitbucket_token=True + ) + assert provider == GitProvider.GITHUB + + def test_invalid_override_raises(self): + with pytest.raises(GitProviderResolutionError, match="not valid"): + resolve_active_git_provider_from_flags( + override="gitlab", has_github_token=True, has_bitbucket_token=False + ) + + +class TestStrategies: + def test_github_default_git_user(self): + assert strategy_for(GitProvider.GITHUB).default_git_user == "x-access-token" + + def test_bitbucket_default_git_user(self): + assert strategy_for(GitProvider.BITBUCKET_CLOUD).default_git_user == "x-token-auth" + + def test_all_strategies_covers_both_providers(self): + providers = {s.provider for s in all_strategies()} + assert providers == {GitProvider.GITHUB, GitProvider.BITBUCKET_CLOUD} + + +class TestSanitizationPatterns: + def _sanitize(self, provider: GitProvider, message: str) -> str: + sanitized = message + for pattern, replacement in strategy_for(provider).sanitization_patterns: + sanitized = pattern.sub(replacement, sanitized) + return sanitized + + def test_github_url_scrubbed(self): + msg = "clone https://x-access-token:ghp_abc123@github.com/org/repo.git failed" + out = self._sanitize(GitProvider.GITHUB, msg) + assert "ghp_abc123" not in out + assert "[REDACTED]" in out + + def test_github_pat_token_scrubbed_standalone(self): + assert "[REDACTED]" in self._sanitize(GitProvider.GITHUB, "token=github_pat_XYZ123") + + def test_bitbucket_url_scrubbed(self): + msg = "clone https://x-token-auth:ATCTT3xFfGN0abc@bitbucket.org/ws/repo.git failed" + out = self._sanitize(GitProvider.BITBUCKET_CLOUD, msg) + assert "ATCTT3xFfGN0abc" not in out + assert "[REDACTED]" in out + + def test_bitbucket_token_scrubbed_standalone(self): + assert "[REDACTED]" in self._sanitize(GitProvider.BITBUCKET_CLOUD, "token=ATCTT3xFfGN0abc_def-2") + + def test_github_patterns_do_not_touch_bitbucket_url(self): + msg = "https://x-token-auth:ATCTT3xFfGN0abc@bitbucket.org/ws/repo.git" + assert self._sanitize(GitProvider.GITHUB, msg) == msg diff --git a/backend/test/services/test_github_auth.py b/backend/test/services/test_github_auth.py index 28dec7e..19fd323 100644 --- a/backend/test/services/test_github_auth.py +++ b/backend/test/services/test_github_auth.py @@ -10,6 +10,7 @@ reset_github_platform_secrets, ) from app.core.workspace_pool_names import DEFAULT_WORKSPACE_POOL +from app.services.git_provider import GitProvider from app.database.memory import InMemoryDatabase from app.services.github_auth import ( GithubAuthResolutionError, @@ -150,6 +151,49 @@ def patched_query(collection, filters=None, **kwargs): assert len(query_calls) == 0, "Should not use raw query for key_uid lookup" +def test_default_pool_uses_bitbucket_token_and_user_when_active(db): + key = Fernet.generate_key() + init_github_platform_secrets_for_tests( + fernet_key=key, + github_token_default=None, + git_user_name_default="x-token-auth", + bitbucket_token_default="bb-platform-token", + active_provider=GitProvider.BITBUCKET_CLOUD, + ) + try: + s = get_holder() + doc = {"workspace_pool": DEFAULT_WORKSPACE_POOL} + ctx = resolve_github_auth_for_api_key_document(doc, s) + assert ctx.token == "bb-platform-token" + assert ctx.git_user_name == "x-token-auth" + finally: + reset_github_platform_secrets() + + +def test_per_key_ciphertext_defaults_git_user_from_active_provider_strategy(db): + """When active=BitBucket and no git_user_name is stored, default to x-token-auth.""" + key = Fernet.generate_key() + init_github_platform_secrets_for_tests( + fernet_key=key, + github_token_default=None, + git_user_name_default=None, + bitbucket_token_default="bb-platform-token", + active_provider=GitProvider.BITBUCKET_CLOUD, + ) + try: + s = get_holder() + ct = s.encrypt_token("per-key-bb-token") + doc = { + "workspace_pool": DEFAULT_WORKSPACE_POOL, + "github_token_ciphertext": ct, + } + ctx = resolve_github_auth_for_api_key_document(doc, s) + assert ctx.token == "per-key-bb-token" + assert ctx.git_user_name == "x-token-auth" + finally: + reset_github_platform_secrets() + + def test_github_cli_env_for_generation(db, secrets): db.set(COL_GENERATION_SESSIONS, "gen-1", { "generation_id": "gen-1", diff --git a/backend/test/services/test_workspace_pool.py b/backend/test/services/test_workspace_pool.py index 446f9db..5d18be4 100644 --- a/backend/test/services/test_workspace_pool.py +++ b/backend/test/services/test_workspace_pool.py @@ -908,3 +908,27 @@ async def test_cleanup_workspace_wrong_state_raises(self, workspace_pool, db): }) with pytest.raises(WorkspacePoolError, match="expected 'cleaning'"): await workspace_pool.cleanup_workspace("ws-01-1") + + +class TestSanitizeTokenInMessageMultiProvider: + """_sanitize_token_in_message scrubs every provider's patterns, not just GitHub's.""" + + def test_github_url_still_scrubbed(self): + msg = "fatal: https://x-access-token:ghp_abcDEF123@github.com/org/repo.git" + out = WorkspacePoolService._sanitize_token_in_message(msg) + assert "ghp_abcDEF123" not in out + assert "[REDACTED]" in out + + def test_bitbucket_url_scrubbed(self): + msg = "fatal: https://x-token-auth:ATCTT3xFfGN0abc123@bitbucket.org/ws/repo.git" + out = WorkspacePoolService._sanitize_token_in_message(msg) + assert "ATCTT3xFfGN0abc123" not in out + assert "[REDACTED]" in out + + def test_bitbucket_token_shape_scrubbed_standalone(self): + out = WorkspacePoolService._sanitize_token_in_message("token=ATCTT3xFfGN0abc_def-2") + assert "[REDACTED]" in out + + def test_explicit_token_still_takes_priority(self): + out = WorkspacePoolService._sanitize_token_in_message("leaked my-secret-value here", token="my-secret-value") + assert "my-secret-value" not in out diff --git a/docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md b/docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md new file mode 100644 index 0000000..cbe7a16 --- /dev/null +++ b/docs/plans/workspace-repos/bitbucket-pr1-commit-and-read.md @@ -0,0 +1,230 @@ +# BitBucket Cloud support — commit, read (P10Y), and auto-provision + +> Standalone, **fully working** first delivery for BitBucket. It unlocks the +> prioritized capability end-to-end: SpecFlow agents clone/commit/push to a +> BitBucket Cloud repo, P10Y reads those commits, and the repos are +> **auto-provisioned** (no manual repo creation). CI/CD (deploy/E2E), +> Server/Data-Center, and the full `git_providers/` refactor + `github_*`→`git_*` +> renames remain out of scope (see `bitbucket-support.md`). +> +> Delivered as **two phases in one shippable change** — Phase A (credentials + +> runtime), Phase B (auto-provisioning). Both land together so the result is a +> working deployment, not a half-path. + +## ⚠️ Core assumption — a single active provider, no mixing + +**A deployment is either all-GitHub or all-BitBucket. Never mixed.** All +workspaces in the pool share one git host. This is a hard assumption and it +simplifies everything: the provider is resolved **once, globally**, from which +token is configured — not per workspace, not per key. There is no coexistence +path to build or test. + +- Active provider = explicit `GIT_PROVIDER` setting if set; else inferred from + **exactly one** of `GITHUB_TOKEN` / `BITBUCKET_TOKEN` being present. +- Both set with no explicit `GIT_PROVIDER`, or neither set → **startup error** + (ambiguous / unconfigured). Fail fast, don't guess. +- A GitHub-only deployment (only `GITHUB_TOKEN` set) resolves to GitHub and is + byte-identical to today — full back-compat, zero migration. + +## The key realization (why this is small) + +The **clone/commit/push runtime and the entire P10Y commit-reading path are +already provider-neutral**. Almost nothing that moves git data is +GitHub-specific: + +- `WorkspacePool._get_authenticated_repo_url` (`workspace_pool.py:1377`) splices + `https://{git_user_name}:{token}@{rest}` into **any** https URL — no + `github.com` anywhere. With `git_user_name="x-token-auth"` and a BitBucket URL + it already emits the exact Cloud access-token form + `https://x-token-auth:{token}@bitbucket.org/{ws}/{repo}.git`. +- All commit/push/archive go through `run_git` on `origin` (`git_utils.py`) — + host-agnostic. +- `GithubAuthContext` (`github_auth.py:16`) is just `(git_user_name, token)`; + Fernet encrypt/decrypt is opaque. +- **P10Y needs zero code change.** It reads commits two ways, both neutral: + local `git log --format=%H\t%s` (`p10y_lib.py:266`), and the external Compass + API keyed by an **integer** `repository_id` (`p10y_api_client.py`). The + `GitType` enum is dead code. Compass supports BitBucket natively; connecting it + to the BitBucket repo is a **user setup step** (Compass side, outside SpecFlow) + — same as the GitHub case today. SHAs match because both read the same repo. + +So the real GitHub-specificity to address is just: credential selection, +`git_user_name`, token log-sanitization, and repo provisioning. + +## User experience (paste one token) + +In the TUI / `.env`, below `GITHUB_TOKEN`, add `BITBUCKET_TOKEN` and +`BITBUCKET_WORKSPACE`. A BitBucket deployment sets those (and leaves +`GITHUB_TOKEN`/`GITHUB_ORG` unset); a GitHub deployment does the reverse. The +user pastes a **BitBucket Cloud Repository/Workspace Access Token** plus the +workspace slug — **no username** (access tokens always use the fixed +`x-token-auth` actor). Run the provisioning script; workspaces are created on +BitBucket and generation runs. That's it. + +--- + +## Phase A — credentials + runtime + +### 1. Minimal provider abstraction (new module) +`backend/app/services/git_provider.py` — one small file (the full +`git_providers/` package split is a later refactor PR): + +```python +class GitProvider(str, Enum): + GITHUB = "github" + BITBUCKET_CLOUD = "bitbucket_cloud" + +@dataclass(frozen=True) +class GitHostStrategy: + provider: GitProvider + default_git_user: str # x-access-token / x-token-auth + def sanitization_patterns(self) -> list[tuple[re.Pattern, str]]: ... + +_GITHUB = GitHostStrategy(GitProvider.GITHUB, "x-access-token", ...) # ghp_/gh[ps]_/github_pat_/@github.com +_BITBUCKET = GitHostStrategy(GitProvider.BITBUCKET_CLOUD, "x-token-auth", ...) # ATCTT…/x-token-auth:/@bitbucket.org + +def strategy_for(provider: GitProvider) -> GitHostStrategy: ... +def all_strategies() -> list[GitHostStrategy]: ... +def resolve_active_git_provider(settings: Settings) -> GitProvider: ... # the global switch (see assumption) +``` +- Move the existing GitHub regexes verbatim into `_GITHUB.sanitization_patterns()`; + add a golden test asserting GitHub scrubbing output is byte-identical to today. +- `resolve_active_git_provider` implements the exactly-one-token rule above. + +### 2. Settings — `BITBUCKET_TOKEN` + explicit override (mirrors GitHub) +`config.py` after line 197: +```python +BITBUCKET_TOKEN_DEFAULT: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("BITBUCKET_TOKEN_DEFAULT", "BITBUCKET_TOKEN"), +) +BITBUCKET_WORKSPACE: Optional[str] = None +GIT_PROVIDER: Optional[str] = None # explicit override; else inferred from token +K8S_SECRET_KEY_BITBUCKET_DEFAULT: str = "bitbucket-token-default" +``` +The `BITBUCKET_TOKEN` alias means the TUI/`.env` value flows in with no other +wiring — exactly how `GITHUB_TOKEN` → `GITHUB_TOKEN_DEFAULT` works today (199). + +### 3. Platform secrets — hold the active provider's token +`github_platform_secrets.py`: +- Add `bitbucket_token_default: Optional[str]` and store the resolved + `active_provider: GitProvider` on `GithubPlatformSecrets`. +- `_load_from_env` (116): read `settings.BITBUCKET_TOKEN_DEFAULT`; compute + `active_provider = resolve_active_git_provider(settings)` (fails fast on + ambiguous/unconfigured). +- `_build_secrets_from_map` (95) + `init_...` (132): read new K8s field + `settings.K8S_SECRET_KEY_BITBUCKET_DEFAULT`; same active-provider resolution. +- Add `active_default_token` / `active_git_user_default` helpers returning the + right values for the active provider. + +### 4. Auth resolution — use the active provider +`github_auth.py` `resolve_github_auth_for_api_key_document` (28): +- **git_user_name default**: when not explicitly set on the key doc, use + `strategy_for(secrets.active_provider).default_git_user` (BitBucket → + `x-token-auth`) instead of the hardcoded `"x-access-token"` (line 47). +- **default-pool token** (58): use `secrets.active_default_token` instead of the + hardcoded `github_token_default`; error message names the active provider. +- Per-key ciphertext path (42) is unchanged (opaque token); only its git_user + default becomes provider-derived. No signature change needed since the active + provider is global (read from `secrets`). + +### 5. Token sanitization — apply all providers (load-bearing safety) +`workspace_pool.py:157` `_sanitize_token_in_message`: keep the explicit `token` +replacement (line 179, host-agnostic, already covers BitBucket), then loop over +`all_strategies()` applying each provider's patterns instead of the three +hardcoded GitHub `re.sub` calls (183–199). Apply **all** providers' patterns +(cheap; defends even if a BitBucket token ever appears in a GitHub deployment's +logs). BitBucket patterns: `@bitbucket\.org` host anchor, `x-token-auth:` +prefix, `ATCTT[A-Za-z0-9_=\-]+` token shape. + +--- + +## Phase B — auto-provisioning (BitBucket Cloud REST) + +`backend/scripts/create_generation_session_repos.py`: + +- Add a `BitbucketCloudAPIClient` (parallel to the existing `GitHubAPIClient` at + `:102`), base `https://api.bitbucket.org/2.0`, header + `Authorization: Bearer {access_token}`: + - `create_repository(workspace, repo_slug)` → `POST /repositories/{workspace}/{repo_slug}` + body `{"scm": "git", "is_private": true}`; treat a 400 "already exists" as + idempotent (mirrors GitHub `repository_exists` guard). + - `repository_exists` → `GET /repositories/{workspace}/{repo_slug}` (200 = exists). + - **No team/access-grant step** — the access token's owner already has write + to repos it creates in the workspace (unlike the GitHub `--team` grant). +- Add `--git-provider {github,bitbucket_cloud}` (default: auto from configured + token via `resolve_active_git_provider`; error if ambiguous) and + `--bitbucket-workspace` (env `BITBUCKET_WORKSPACE`). +- Build `repo_url` from the provider instead of the hardcoded github.com at + 707/776: `https://bitbucket.org/{workspace}/{repo_slug}` for BitBucket. Emit it + into the workspace doc via the existing `create_workspace_document` path (629) + and the `--output-workspace-config` path. +- **P10Y metrics-enable is skipped for BitBucket** (that step hits GitHub-side + registration). `p10y_repository_id` is set externally when Compass registers + the BitBucket repo — treat BitBucket provisioning like the existing + `--skip-metrics` path and let `p10y_repository_id` come from config (or null). + +--- + +## TUI / onboarding + +`mcp_server/tui/config.py`: add `BITBUCKET_TOKEN`, `BITBUCKET_WORKSPACE` to +`ENV_SECRET_KEYS` (32) and `BITBUCKET_TOKEN` to `MASKED_KEYS` (43). Add matching +entries to `.env.quickstart.example`. The load-time parity assertion stays green. +`mcp_server/tui/onboarding.py`: present GitHub **or** BitBucket as the git step +(a provider choice, honoring the no-mixing assumption), collecting either +`GITHUB_TOKEN`/`GITHUB_ORG` or `BITBUCKET_TOKEN`/`BITBUCKET_WORKSPACE`. BitBucket +wording: "paste a BitBucket Cloud access token — no username needed." + +## P10Y — no code change +Confirmed neutral. Optionally add `BITBUCKET = "bitbucket"` to the dead `GitType` +enum for future clarity (no behavior change). Compass's BitBucket connection is a +user setup step (Compass supports it natively) — note it in the docs/PR +description as a prerequisite, just like the GitHub connection is today. + +## Persistence / back-compat (zero migration) +- No per-workspace `git_provider` field is needed — provider is global. Existing + workspace/api-key docs are read unchanged. +- All new `Settings`/secret fields default to `None`; a GitHub-only deployment + (only `GITHUB_TOKEN` set) resolves to GitHub and behaves identically. +- API-key doc unchanged (still `github_token_ciphertext`); the ciphertext is an + opaque BitBucket token in a BitBucket deployment. + +## Explicitly out of scope (later PRs) +- CI/CD (Pipelines vs Actions), deploy/E2E — `gh` CLI, `GH_TOKEN`, + `github_cli_env_for_generation`, deploy prompt recipes. Riskiest; last. +- BitBucket **Server/Data Center** (clone path `/scm/…`, arbitrary host, no + managed CI). +- Full `git_providers/` package split (hosting/provisioning/deploy ABCs) and the + `github_*`→`git_*` module/field renames + `PUT /auth/git-token` route alias. +- **Per-key / mixed-provider** support — explicitly excluded by the no-mixing + assumption. +- Notification web-URL scheme (`/tree/` vs `/src/`) — cosmetic; can ride along. + +## Tests +- Golden: GitHub sanitization + auth resolution byte-identical to current. +- `resolve_active_git_provider`: only-GitHub → GITHUB; only-BitBucket → CLOUD; + both without `GIT_PROVIDER` → error; neither → error; explicit `GIT_PROVIDER` + wins. +- Sanitizer: `ATCTT…`, `x-token-auth:@bitbucket.org` URL, explicit-token all + scrubbed; GitHub cases unchanged. +- Auth: default-pool with active=BitBucket yields git_user `x-token-auth` + the + BitBucket token; active=GitHub unchanged. +- Clone flow with active=BitBucket (mock `run_git`): authenticated URL is + `https://x-token-auth:{token}@bitbucket.org/...`. +- Provisioning: `BitbucketCloudAPIClient` create/exists with mocked `httpx`; + `--git-provider bitbucket_cloud --dry-run` emits a bitbucket.org `repo_url`; + idempotent create on 400-already-exists. +- `mcp_server` config parity assertion green with the new keys. + +Run `make unit-tests` (record baseline first; pass count ≥ baseline). Do **not** +use `make integration-tests`. `make check-complexity-diff` after (touches +`workspace_pool.py` and the provisioning script). + +## End-to-end sanity +Set `BITBUCKET_TOKEN` + `BITBUCKET_WORKSPACE` (leave GitHub vars unset), run +`create_generation_session_repos.py --git-provider bitbucket_cloud`, confirm +repos created on bitbucket.org and workspace docs carry bitbucket URLs. Run a +generation: clone → agents commit → `git push origin main` succeeds; P10Y's +local `git log` allowlist picks up the SHAs. Verify a GitHub-only deployment is +byte-identical (golden tests green). diff --git a/docs/plans/workspace-repos/bitbucket-support.md b/docs/plans/workspace-repos/bitbucket-support.md new file mode 100644 index 0000000..0c288b3 --- /dev/null +++ b/docs/plans/workspace-repos/bitbucket-support.md @@ -0,0 +1,92 @@ +# Add BitBucket Support to SpecFlow + +## Context + +SpecFlow is hardwired to **GitHub** end-to-end with **no git-hosting-provider abstraction**. GitHub-specific logic is scattered across credential resolution, authenticated-URL construction, token sanitization, repo provisioning, deploy prompts (GitHub Actions + `gh` CLI), onboarding, and notifications. The only pre-existing notion of provider variance is a dormant `GitType` enum (`GITHUB`/`GITLAB`) in the P10Y models. `bitbucket` appears nowhere except the branch name — this is greenfield. + +We need to add BitBucket as a first-class git host. **Scope decisions (confirmed with user):** +1. **Full parity**: git hosting AND CI/CD (deploy/E2E). +2. **Both BitBucket Cloud (bitbucket.org) and Server/Data Center (self-hosted).** +3. **Auth = Access Tokens** (Repository/Workspace access tokens), not app passwords. +4. **Per-generation/per-key coexistence**: GitHub and BitBucket usable side-by-side; provider resolved per workspace / per API-key, mirroring the existing per-key encrypted-token model. + +**Outcome**: a `GitProvider` abstraction (Open/Closed + SRP) that generalizes the three currently-hardcoded single-source mechanisms — authenticated-URL builder, token sanitizer, deploy recipe — *in place*, plus new BitBucket implementations, so a workspace on any provider can clone/commit/push/archive, be provisioned, and (Cloud) deploy via Pipelines. + +## ⚠️ Design decision to confirm during execution: Server/DC CI/CD + +BitBucket **Server/Data Center has no native CI** (no Pipelines). "Full CI/CD parity" is achievable only for **Cloud** (Pipelines via REST API). For **Server/DC**, the recommended design **degrades gracefully**: git-hosting + codegen fully work, but the deploy/E2E loop is **skipped with a structured USER NOTICE** (reusing the existing DEPLOY FAILURE REPORT / Slack channel) telling the operator that Server/DC deploy needs external CI (Jenkins/Bamboo) SpecFlow doesn't own. A future `GenericWebhookDeployStrategy` (hitting a configured `DEPLOY_WEBHOOK_URL`) can slot behind the same interface without reopening the work. This plan proceeds on that basis. + +## The Abstraction + +New package `backend/app/services/git_providers/`, SRP-split across the three layers it touches (git subprocess, provisioning script, deploy/prompt). One `Enum` selects implementations via a registry. + +- `enums.py` — `GitProvider(str, Enum)`: `GITHUB`, `BITBUCKET_CLOUD`, `BITBUCKET_SERVER`. +- `hosting.py` — `GitHostingProvider(ABC)`: `authenticated_clone_url(repo_url, auth)`, `token_sanitization_patterns()`, `branch_web_url(repo_url, branch)`, `default_git_user()`. +- `provisioning.py` — `RepoProvisioner(ABC)`: `repository_exists`, `create_repository`, `grant_write_access`, `get_authenticated_actor`, `web_url_for`. +- `deploy_strategy.py` — `DeployStrategy(ABC)`: `supports_managed_ci` (ClassVar bool), `deploy_env(token)`, `build_deploy_recipe(ctx)`, `unsupported_ci_notice(ctx)`. +- `registry.py` — `hosting_for`, `deploy_strategy_for`, `provisioner_for`, `provider_from_repo_url(repo_url)` (host-based default: `bitbucket.org`→CLOUD, `github.com`→GITHUB, else SERVER-requires-explicit-config). +- `github.py`, `bitbucket.py` — concrete impls. + +Provider-neutral pieces to **REUSE unchanged**: `git_utils.run_git`, `git_archive_service.GitArchiveService` (operate on `origin`), per-workspace `repo_url` storage. + +**Persistence / back-compat** (must read existing data with zero migration required): +- Workspace doc gains `git_provider` (nullable; read fallback = `provider_from_repo_url(repo_url)` → `github` for existing URLs). Server/DC workspaces MUST carry it explicitly (host isn't inferable). +- API-key doc: read ciphertext new→legacy (`git_token_ciphertext` → `github_token_ciphertext`); write only new. Add `git_provider` on key doc (default `github` for legacy). Keep `PUT /api/v1/auth/github-token` route + add alias `PUT /api/v1/auth/git-token`; thin re-export shim for renamed modules for one release. + +## Phased Implementation + +Order deliberately lands **git-hosting before CI/CD** (CI/CD is riskiest). + +### Phase 0 — Scaffolding + GitHub-first (no behavior change) +Create the `git_providers/` package + interfaces + `GitHubProvider` that *moves* today's exact logic (the `user:token@` URL builder and `ghp_`/`ghs_`/`github_pat_`+`@github.com` regexes out of `workspace_pool.py`; the `{"GH_TOKEN": token}` env and `gh workflow run` recipe out of `agents_claude_code.py`). +- **Golden/characterization tests** (`backend/tests/unit/git_providers/test_github_provider.py`) assert byte-identical output vs current behavior — makes Phases 2 & 5 provably non-regressive. + +### Phase 1 — Persistence + auth/secret generalization +- Rename `backend/app/core/github_platform_secrets.py` → `git_platform_secrets.py` (keep re-export shim). `GithubPlatformSecrets` → `GitPlatformSecrets` holding Fernet + `dict[GitProvider, DefaultIdentity]`. New settings `GIT_TOKEN_DEFAULT_BITBUCKET` + K8s key `K8S_SECRET_KEY_BITBUCKET_DEFAULT`; keep `GITHUB_TOKEN_DEFAULT`/`GITHUB_TOKEN` aliases. +- `backend/app/services/github_auth.py` → generalize to `git_auth.py`: `GithubAuthContext` → `GitAuthContext(git_user_name, token, provider)`. `resolve_git_auth_for_api_key_document(doc, workspace_provider, secrets)` preserves order (per-key ciphertext → per-provider default → error). `github_cli_env_for_generation` → `deploy_env_for_generation` returning `strategy.deploy_env(token)`. Keep old names as deprecated aliases to stage callsite churn. +- Modify `backend/app/core/config.py` (new settings/K8s keys + aliases), `backend/app/core/app_lifecycle.py` (renamed init), `backend/app/api/v1/auth.py` (`GitTokenUploadBody`, write `git_token_ciphertext`+`git_provider`, accept old shape). +- Idempotent, read-safe migration `backend/scripts/migrate_git_provider.py` (backfill workspace `git_provider`; copy ciphertext field). Not required for reads. +- Tests: auth-resolution matrix (legacy-only / new-only / per-provider default / error), secrets loader from env + mocked K8s map, both providers. + +### Phase 2 — Clone / push / sanitize generalization (git-hosting parity lands) +- `backend/app/services/workspace_pool.py`: `_get_authenticated_repo_url` delegates to `hosting_for(auth.provider).authenticated_clone_url(...)` (delete inline string surgery). `_sanitize_token_in_message` keeps explicit-token replacement, then applies patterns from **all** registered providers (defensive — never leak even on misresolution). Every `resolve_github_auth_for_generation_id(...)` callsite → `resolve_git_auth_for_generation_id(...)` (loads workspace `git_provider`). Clone/init/archive/reset funcs otherwise unchanged (operate on `origin`/`run_git`). GitHub-specific auth-failure hint string becomes provider-parameterized. +- Create `backend/app/services/git_providers/bitbucket.py`: `BitbucketCloudProvider` (clone `https://x-token-auth:{token}@bitbucket.org/{ws}/{repo}.git`; branch `/src/{branch}`; sanitize `@bitbucket.org`, `x-token-auth:`, `ATCTT` prefix) and `BitbucketServerProvider` (clone `https://x-token-auth:{token}@{host}/scm/{proj}/{repo}.git`; branch `/browse?at=refs%2Fheads%2F{branch}`; host from `BITBUCKET_SERVER_BASE_URL`). +- Tests: URL injection + idempotence, BB sanitization shapes, branch-url builders, clone flow with BB workspace doc (mock `run_git`). + +### Phase 3 — Repo provisioning +- `backend/scripts/create_generation_session_repos.py`: extract `GitHubAPIClient` to `RepoProvisioner` (in `github.py`); add `BitbucketCloudProvisioner` (`https://api.bitbucket.org/2.0`, `POST /repositories/{ws}/{repo}`, permission endpoints, `Bearer` access token) and `BitbucketServerProvisioner` (`{base}/rest/api/1.0/projects/{key}/repos`). Add `--git-provider` flag; generalize `--github-org`/`--team` → `--namespace`/`--principal` (keep aliases). Emit `git_provider` + `provisioner.web_url_for(name)` as `repo_url` (no hardcoded `github.com`). `_normalize_git_url` unchanged. +- Tests: provisioner unit tests (mock `httpx`) per provider; workspace-doc emission. + +### Phase 4 — P10Y + notifications (small, low-risk) +- `backend/app/services/p10y/p10y_api_models.py`: add `GitType.BITBUCKET = "bitbucket"`; set `RepositoryDetails.git_provider` for BB repos. +- `backend/app/core/notifications.py` (~line 1144): replace hardcoded `{repo_url}/tree/{branch}` with `hosting_for(provider).branch_web_url(...)` (resolve provider from workspace doc). Third single-source site — generalize, don't branch beside it. + +### Phase 5 — CI/CD layer (riskiest — last) +Bifurcation: **GitHub** = existing `gh` recipe (now inside `GitHubProvider`); **BB Cloud** = `curl`+`jq` recipe (`POST /2.0/repositories/{ws}/{repo}/pipelines/` custom selector, poll `GET .../pipelines/{uuid}` for `state.name==COMPLETED`, config `bitbucket-pipelines.yml`) — curl+jq already in `bash_usage`, no new tooling; **BB Server/DC** = `supports_managed_ci=False`, deploy skipped with USER NOTICE (see top-of-plan design decision). +- `backend/app/schemas/deploy_context.py`: `DeployGithubContext` → `DeployContext(repo, ref, deploy_workflow, provider)` (keep alias). +- `backend/app/services/workflow_steps.py` `_build_deploy_github_context` (~1064): read workspace `git_provider`; if `not supports_managed_ci`, emit notice + skip (generalize the `github_repo` guard at ~1176 to capability+repo). +- `backend/app/prompts/agents_claude_code.py` `generate_deploy_phase_agent_template` (~1139): replace the hardcoded GitHub-Actions bash block with `strategy.build_deploy_recipe(ctx)` (GitHub block moves verbatim into `GitHubDeployStrategy`; BB Cloud block new). Planning-side text (~741) becomes provider-agnostic. +- `backend/app/services/claude_code.py` (~1178): `deploy_extra_env = deploy_env_for_generation(...)`. +- `backend/app/core/tool_usage.py`: add `deploy_extra_tools_for(provider)` gating `Bash(gh:*)` to GitHub deploys (BB uses curl/jq). +- `backend/app/services/agent_hooks.py` `_ci_blocklist`: verify BB `for poll in seq` loop isn't caught by `_watch_flag_blocklist`; add a `gh run watch` analogue only if a BB hang command exists. +- `backend/app/standards/deployment_standards.md`: add BB Pipelines section paralleling GitHub Actions; default artifacts gain `bitbucket-pipelines.yml`; note Server/DC has no managed CI. +- Tests: `build_deploy_recipe` golden per provider; `supports_managed_ci=False` produces notice + skips; `deploy_env` var; `deploy_extra_tools_for` gating. + +### Phase 6 — Onboarding / TUI / config / docs +- `mcp_server/tui/onboarding.py`: generalize hardcoded `_GITHUB` step into a provider-selecting step (GitHub / BB Cloud / BB Server) collecting `GIT_TOKEN`, namespace (org/workspace/project), `GIT_USER_NAME`, Server base URL. Mirror the LLM `_PROVIDER` pattern. +- `mcp_server/tui/config.py`: register new keys in `ENV_SECRET_KEYS`/`MASKED_KEYS` (load-time parity assertion is a hard gate). +- `.env.quickstart.example`, `QUICKSTART.md`, `specflow-init.sh` (`_GH_ORG` → provider-parameterized): add BB variants. +- MCP forwarding stays provider-agnostic (provider resolved server-side per workspace/key); forward a `GIT_PROVIDER` hint only if onboarding must set a per-key default. +- Tests: `mcp_server` config parity assertion; onboarding validation per provider. + +## Biggest Risks +1. **Server/DC has no managed CI** — graceful degradation, not a fake pipeline. Needs sign-off that this satisfies requirement #1 for Server/DC (see top-of-plan). +2. **Exact BB access-token clone/REST forms** differ Cloud vs Server (`x-token-auth`; `/scm/` path; `Bearer` REST). Isolated by the abstraction but need live validation against a real BB instance — `authenticated_clone_url` for Server is highest-uncertainty. +3. **Token-leak prevention is a security control** — apply all providers' sanitization patterns; Phase 0 golden tests must prove GitHub scrubbing unchanged; BB patterns (`x-token-auth:`, `ATCTT…`) must be added or BB tokens leak into logs/Slack. +4. **Field-rename back-compat** — `github_token_ciphertext` + `github-token` route are in production data/clients; read-fallback + route alias + one-release shim + idempotent non-destructive migration are mandatory. +5. **Deploy recipe is autonomous prompt text (up to 30 min)** — BB Cloud curl/jq poll loop must have the same hard timeout + non-fatal-403 semantics as GitHub and must not trip `_ci_blocklist`, else deploys hang mid-run with no human in the loop. + +## Verification +- `make unit-tests` after each phase; pass count ≥ baseline (record baseline first). New `backend/tests/unit/git_providers/` package: one module per provider per interface + registry/host-derivation test. Clone/deploy flow tests use in-memory DB + mocked `run_git`/`httpx` (no network). Do NOT use `make integration-tests` for these. +- `make check-complexity-diff` after Phases 2 and 5 (the two that refactor large existing files). +- End-to-end sanity: run `create_generation_session_repos.py --git-provider bitbucket_cloud --dry-run` to confirm provisioning wiring; exercise a BB Cloud workspace through clone→commit→push→archive with a real (or sandbox) BB access token; confirm the deploy recipe triggers a Pipelines custom run and polls to completion. Confirm GitHub path is byte-identical (golden tests green) — no regression. diff --git a/mcp_server/tests/test_tui_app.py b/mcp_server/tests/test_tui_app.py index 6939642..610c3b1 100644 --- a/mcp_server/tests/test_tui_app.py +++ b/mcp_server/tests/test_tui_app.py @@ -378,9 +378,9 @@ async def test_writes_chosen_provider_key_and_runs_init(self, tmp_path): assert screen.current_step_id == "provider" await self._set(screen, "OPENROUTER_API_KEY", "or-key") - await pilot.press("ctrl+n") # -> github + await pilot.press("ctrl+n") # -> git (default choice: GitHub) await pilot.pause() - assert screen.current_step_id == "github" + assert screen.current_step_id == "git" await self._set(screen, "GITHUB_TOKEN", "tok") await pilot.press("ctrl+n") # -> compass @@ -414,11 +414,11 @@ async def test_anthropic_path_writes_only_anthropic(self, tmp_path): await pilot.press("ctrl+n") # -> provider await pilot.pause() # Select the Anthropic radio (second option) and fill its key. - buttons = list(screen.query("#onboard-provider RadioButton").results()) + buttons = list(screen.query("#onboard-choice RadioButton").results()) buttons[1].value = True # toggles selection, fires RadioSet.Changed await pilot.pause() await self._set(screen, "ANTHROPIC_API_KEY", "ant-key") - await pilot.press("ctrl+n") # -> github + await pilot.press("ctrl+n") # -> git (default choice: GitHub) await pilot.pause() await self._set(screen, "GITHUB_TOKEN", "tok") await pilot.press("ctrl+n") # -> compass @@ -433,6 +433,40 @@ async def test_anthropic_path_writes_only_anthropic(self, tmp_path): assert "ANTHROPIC_API_KEY=ant-key" in env assert "OPENROUTER_API_KEY=" not in env + @pytest.mark.asyncio + async def test_bitbucket_path_writes_only_bitbucket_fields(self, tmp_path): + run_init = AsyncMock(return_value=0) + a, b, c, d, e, f = self._gate_patches(tmp_path, run_init) + with a, b, c, d, e, f: + app = tui_app.SpecFlowTUI(root=tmp_path, generation_id=None, poll_interval=999) + async with app.run_test() as pilot: + await pilot.pause() + screen = app.screen + await pilot.press("ctrl+n") # -> provider + await pilot.pause() + await self._set(screen, "OPENROUTER_API_KEY", "or-key") + await pilot.press("ctrl+n") # -> git + await pilot.pause() + assert screen.current_step_id == "git" + # Select the BitBucket radio (second option) and fill its fields. + buttons = list(screen.query("#onboard-choice RadioButton").results()) + buttons[1].value = True # toggles selection, fires RadioSet.Changed + await pilot.pause() + await self._set(screen, "BITBUCKET_TOKEN", "bb-tok") + await self._set(screen, "BITBUCKET_WORKSPACE", "my-ws") + await pilot.press("ctrl+n") # -> compass + await pilot.pause() + await self._set(screen, "P10Y_API_KEY", "p10y") + await pilot.press("ctrl+n") # -> review + await pilot.pause() + await pilot.press("ctrl+s") + await pilot.pause() + run_init.assert_awaited_once() + env = (tmp_path / ".env").read_text() + assert "BITBUCKET_TOKEN=bb-tok" in env + assert "BITBUCKET_WORKSPACE=my-ws" in env + assert "GITHUB_TOKEN=" not in env + @pytest.mark.asyncio async def test_validation_blocks_advance_without_required(self, tmp_path): run_init = AsyncMock(return_value=0) @@ -466,12 +500,12 @@ async def test_back_preserves_entered_value(self, tmp_path): await pilot.press("ctrl+n") # -> provider await pilot.pause() await self._set(screen, "OPENROUTER_API_KEY", "or-key") - await pilot.press("ctrl+n") # -> github + await pilot.press("ctrl+n") # -> git await pilot.pause() await self._set(screen, "GITHUB_ORG", "my-org") await pilot.press("ctrl+b") # back -> provider await pilot.pause() - await pilot.press("ctrl+n") # forward -> github + await pilot.press("ctrl+n") # forward -> git await pilot.pause() assert screen.query_one("#onb-GITHUB_ORG", Input).value == "my-org" @@ -484,11 +518,11 @@ async def test_secret_field_is_masked(self, tmp_path): async with app.run_test() as pilot: await pilot.pause() screen = app.screen - # Advance to the GitHub step which carries a masked + a plain field. + # Advance to the git step (default GitHub choice) which carries a masked + a plain field. await pilot.press("ctrl+n") # provider await pilot.pause() await self._set(screen, "OPENROUTER_API_KEY", "or-key") - await pilot.press("ctrl+n") # github + await pilot.press("ctrl+n") # git await pilot.pause() assert screen.query_one("#onb-GITHUB_TOKEN", Input).password is True assert screen.query_one("#onb-GIT_USER_NAME", Input).password is False diff --git a/mcp_server/tests/test_tui_onboarding.py b/mcp_server/tests/test_tui_onboarding.py index 4a2a152..fda5ae9 100644 --- a/mcp_server/tests/test_tui_onboarding.py +++ b/mcp_server/tests/test_tui_onboarding.py @@ -6,6 +6,8 @@ from tui import config, onboarding from tui.onboarding import ( + GIT_PROVIDER_BITBUCKET, + GIT_PROVIDER_GITHUB, PROVIDER_ANTHROPIC, PROVIDER_OPENROUTER, StepKind, @@ -16,16 +18,21 @@ def _all_fields(): fields = [] for step in onboarding.STEPS: fields.extend(step.fields) - fields.extend(c.field for c in step.choices) + for choice in step.choices: + fields.extend(choice.fields) return fields +def _chosen(provider=PROVIDER_OPENROUTER, git=GIT_PROVIDER_GITHUB): + return {"provider": provider, "git": git} + + class TestStepContent: def test_step_order(self): assert [s.step_id for s in onboarding.STEPS] == [ "welcome", "provider", - "github", + "git", "compass", "review", ] @@ -42,86 +49,158 @@ def test_provider_step_defaults_to_openrouter(self): assert onboarding.PROVIDER_STEP.default_choice == PROVIDER_OPENROUTER assert onboarding.PROVIDER_STEP.kind is StepKind.CHOICE - def test_optional_identity_fields_not_required(self): - github = next(s for s in onboarding.STEPS if s.step_id == "github") - by_key = {f.key: f for f in github.fields} + def test_git_step_defaults_to_github(self): + assert onboarding.GIT_STEP.default_choice == GIT_PROVIDER_GITHUB + assert onboarding.GIT_STEP.kind is StepKind.CHOICE + + def test_choice_steps_are_provider_and_git(self): + assert [s.step_id for s in onboarding.CHOICE_STEPS] == ["provider", "git"] + + def test_default_choices(self): + assert onboarding.default_choices() == { + "provider": PROVIDER_OPENROUTER, + "git": GIT_PROVIDER_GITHUB, + } + + def test_github_optional_identity_fields_not_required(self): + fields = onboarding.choice_fields(onboarding.GIT_STEP, GIT_PROVIDER_GITHUB) + by_key = {f.key: f for f in fields} assert by_key["GITHUB_TOKEN"].required is True assert by_key["GITHUB_ORG"].required is False assert by_key["GIT_USER_NAME"].required is False + def test_bitbucket_fields_both_required(self): + fields = onboarding.choice_fields(onboarding.GIT_STEP, GIT_PROVIDER_BITBUCKET) + by_key = {f.key: f for f in fields} + assert by_key["BITBUCKET_TOKEN"].required is True + assert by_key["BITBUCKET_WORKSPACE"].required is True + class TestRequiredKeys: - def test_openrouter_required_set(self): - assert set(onboarding.required_keys(PROVIDER_OPENROUTER)) == { + def test_openrouter_github_required_set(self): + assert set(onboarding.required_keys(_chosen(PROVIDER_OPENROUTER, GIT_PROVIDER_GITHUB))) == { "OPENROUTER_API_KEY", "GITHUB_TOKEN", "P10Y_API_KEY", } def test_anthropic_required_set(self): - assert set(onboarding.required_keys(PROVIDER_ANTHROPIC)) == { + assert set(onboarding.required_keys(_chosen(PROVIDER_ANTHROPIC, GIT_PROVIDER_GITHUB))) == { "ANTHROPIC_API_KEY", "GITHUB_TOKEN", "P10Y_API_KEY", } + def test_bitbucket_required_set(self): + assert set(onboarding.required_keys(_chosen(PROVIDER_OPENROUTER, GIT_PROVIDER_BITBUCKET))) == { + "OPENROUTER_API_KEY", + "BITBUCKET_TOKEN", + "BITBUCKET_WORKSPACE", + "P10Y_API_KEY", + } + class TestValidateStep: def _step(self, step_id): return next(s for s in onboarding.STEPS if s.step_id == step_id) def test_info_and_review_always_pass(self): - assert onboarding.validate_step(self._step("welcome"), {}, PROVIDER_OPENROUTER) is None - assert onboarding.validate_step(self._step("review"), {}, PROVIDER_OPENROUTER) is None + assert onboarding.validate_step(self._step("welcome"), {}, _chosen()) is None + assert onboarding.validate_step(self._step("review"), {}, _chosen()) is None def test_provider_requires_chosen_key(self): step = self._step("provider") - assert onboarding.validate_step(step, {}, PROVIDER_OPENROUTER) is not None + assert onboarding.validate_step(step, {}, _chosen(PROVIDER_OPENROUTER)) is not None ok = {"OPENROUTER_API_KEY": "k"} - assert onboarding.validate_step(step, ok, PROVIDER_OPENROUTER) is None + assert onboarding.validate_step(step, ok, _chosen(PROVIDER_OPENROUTER)) is None # The unchosen key does not satisfy the chosen provider. - assert onboarding.validate_step(step, ok, PROVIDER_ANTHROPIC) is not None + assert onboarding.validate_step(step, ok, _chosen(PROVIDER_ANTHROPIC)) is not None - def test_github_requires_token_but_not_optionals(self): - step = self._step("github") - assert onboarding.validate_step(step, {}, PROVIDER_OPENROUTER) is not None + def test_git_github_requires_token_but_not_optionals(self): + step = self._step("git") + assert onboarding.validate_step(step, {}, _chosen(git=GIT_PROVIDER_GITHUB)) is not None + assert ( + onboarding.validate_step(step, {"GITHUB_TOKEN": "t"}, _chosen(git=GIT_PROVIDER_GITHUB)) + is None + ) + + def test_git_bitbucket_requires_both_fields(self): + step = self._step("git") + chosen = _chosen(git=GIT_PROVIDER_BITBUCKET) + assert onboarding.validate_step(step, {}, chosen) is not None + assert ( + onboarding.validate_step(step, {"BITBUCKET_TOKEN": "t"}, chosen) is not None + ) assert ( - onboarding.validate_step(step, {"GITHUB_TOKEN": "t"}, PROVIDER_OPENROUTER) is None + onboarding.validate_step( + step, {"BITBUCKET_TOKEN": "t", "BITBUCKET_WORKSPACE": "ws"}, chosen + ) + is None ) + # GitHub fields left over from a prior choice don't satisfy BitBucket. + assert onboarding.validate_step(step, {"GITHUB_TOKEN": "t"}, chosen) is not None class TestValidateAll: - def _complete(self): + def _complete_github(self): return { "OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1", } - def test_complete_passes(self): - assert onboarding.validate_all(self._complete(), PROVIDER_OPENROUTER) is None + def _complete_bitbucket(self): + return { + "OPENROUTER_API_KEY": "or", + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } + + def test_complete_github_passes(self): + assert onboarding.validate_all(self._complete_github(), _chosen()) is None + + def test_complete_bitbucket_passes(self): + assert onboarding.validate_all(self._complete_bitbucket(), _chosen(git=GIT_PROVIDER_BITBUCKET)) is None def test_missing_p10y_fails(self): - vals = self._complete() + vals = self._complete_github() del vals["P10Y_API_KEY"] - assert onboarding.validate_all(vals, PROVIDER_OPENROUTER) is not None + assert onboarding.validate_all(vals, _chosen()) is not None - def test_wrong_provider_key_fails(self): + def test_wrong_llm_provider_key_fails(self): # OpenRouter set but Anthropic chosen → the chosen key is missing. - assert onboarding.validate_all(self._complete(), PROVIDER_ANTHROPIC) is not None + assert onboarding.validate_all(self._complete_github(), _chosen(PROVIDER_ANTHROPIC)) is not None + + def test_wrong_git_provider_fields_fail(self): + # GitHub fields set but BitBucket chosen → the chosen fields are missing. + assert ( + onboarding.validate_all(self._complete_github(), _chosen(git=GIT_PROVIDER_BITBUCKET)) + is not None + ) class TestEnvSatisfiesRequirements: - def test_complete_openrouter_env_passes(self): + def test_complete_openrouter_github_env_passes(self): assert onboarding.env_satisfies_requirements( {"OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1"} ) - def test_complete_anthropic_env_passes(self): + def test_complete_anthropic_github_env_passes(self): assert onboarding.env_satisfies_requirements( {"ANTHROPIC_API_KEY": "an", "GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1"} ) + def test_complete_bitbucket_env_passes(self): + assert onboarding.env_satisfies_requirements( + { + "OPENROUTER_API_KEY": "or", + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } + ) + def test_missing_llm_key_fails(self): assert not onboarding.env_satisfies_requirements( {"GITHUB_TOKEN": "gh", "P10Y_API_KEY": "p1"} @@ -132,6 +211,11 @@ def test_missing_shared_key_fails(self): {"OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh"} ) + def test_bitbucket_missing_workspace_fails(self): + assert not onboarding.env_satisfies_requirements( + {"OPENROUTER_API_KEY": "or", "BITBUCKET_TOKEN": "bb", "P10Y_API_KEY": "p1"} + ) + def test_empty_env_fails(self): assert not onboarding.env_satisfies_requirements({}) @@ -145,7 +229,7 @@ def test_drops_empty_and_unchosen_provider_key(self): "GITHUB_ORG": "", # empty → dropped "P10Y_API_KEY": "p1", } - out = onboarding.collected_secrets(values, PROVIDER_OPENROUTER) + out = onboarding.collected_secrets(values, _chosen(PROVIDER_OPENROUTER)) assert out == { "OPENROUTER_API_KEY": "or", "GITHUB_TOKEN": "gh", @@ -159,6 +243,22 @@ def test_keeps_present_optionals(self): "GITHUB_ORG": "my-org", "P10Y_API_KEY": "p1", } - out = onboarding.collected_secrets(values, PROVIDER_ANTHROPIC) + out = onboarding.collected_secrets(values, _chosen(PROVIDER_ANTHROPIC)) assert out["GITHUB_ORG"] == "my-org" assert "OPENROUTER_API_KEY" not in out + + def test_bitbucket_choice_drops_github_fields(self): + values = { + "OPENROUTER_API_KEY": "or", + "GITHUB_TOKEN": "gh", # left over from a prior choice → dropped + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } + out = onboarding.collected_secrets(values, _chosen(git=GIT_PROVIDER_BITBUCKET)) + assert out == { + "OPENROUTER_API_KEY": "or", + "BITBUCKET_TOKEN": "bb", + "BITBUCKET_WORKSPACE": "ws", + "P10Y_API_KEY": "p1", + } diff --git a/mcp_server/tui/app.py b/mcp_server/tui/app.py index 64244ef..62bbcec 100644 --- a/mcp_server/tui/app.py +++ b/mcp_server/tui/app.py @@ -1179,7 +1179,7 @@ class OnboardingScreen(_SpecFlowScreen): def __init__(self) -> None: super().__init__() self._index = 0 - self._chosen_provider = onboarding.PROVIDER_STEP.default_choice + self._chosen: dict[str, str] = onboarding.default_choices() self._values: dict[str, str] = {} @property @@ -1259,24 +1259,31 @@ def _field_rows(self, fields: tuple[onboarding.Field, ...]) -> list[Any]: return rows def _choice_widgets(self, step: onboarding.Step) -> list[Any]: + chosen_option = self._chosen[step.step_id] radio = RadioSet( *( - RadioButton(c.label, value=(c.option_id == self._chosen_provider)) + RadioButton(c.label, value=(c.option_id == chosen_option)) for c in step.choices ), - id="onboard-provider", + id="onboard-choice", ) - # The selected provider's how-to + key field are (re)built into the + # The selected option's how-to + field(s) are (re)built into the # detail container by _render_choice_detail on mount and on selection. return [radio, VerticalScroll(id="onboard-choice-detail")] def _review_widget(self) -> Any: lines: list[str] = [] - chosen = onboarding.provider_field(self._chosen_provider) - secrets = onboarding.collected_secrets(self._values, self._chosen_provider) - field_by_key = {f.key: f for s in onboarding.STEPS for f in s.fields} - field_by_key[chosen.key] = chosen - for key in [chosen.key, "GITHUB_TOKEN", "GITHUB_ORG", "GIT_USER_NAME", "P10Y_API_KEY"]: + secrets = onboarding.collected_secrets(self._values, self._chosen) + field_by_key: dict[str, onboarding.Field] = {f.key: f for s in onboarding.STEPS for f in s.fields} + review_keys: list[str] = [] + for step in onboarding.STEPS: + if step.kind is onboarding.StepKind.CHOICE: + fields = onboarding.choice_fields(step, self._chosen[step.step_id]) + field_by_key.update({f.key: f for f in fields}) + review_keys.extend(f.key for f in fields) + else: + review_keys.extend(f.key for f in step.fields) + for key in review_keys: f = field_by_key[key] if f.masked: shown = "•••• set" if secrets.get(key) else "— (missing)" @@ -1319,7 +1326,7 @@ def action_back(self) -> None: def action_next(self) -> None: self._capture() - error = onboarding.validate_step(self.current_step, self._values, self._chosen_provider) + error = onboarding.validate_step(self.current_step, self._values, self._chosen) if error: self.notify(error, severity="error") return @@ -1342,7 +1349,8 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self._save_and_init() def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - self._chosen_provider = onboarding.PROVIDER_STEP.choices[event.index].option_id + step = self.current_step + self._chosen[step.step_id] = step.choices[event.index].option_id self.call_later(self._render_choice_detail) async def _render_choice_detail(self) -> None: @@ -1351,23 +1359,22 @@ async def _render_choice_detail(self) -> None: except NoMatches: return await detail.remove_children() - choice = next( - c for c in onboarding.PROVIDER_STEP.choices if c.option_id == self._chosen_provider - ) + step = self.current_step + choice = next(c for c in step.choices if c.option_id == self._chosen[step.step_id]) widgets: list[Any] = [Static(choice.why, classes="onboard-why")] widgets.extend(self._howto_widgets(choice.how_to)) - widgets.extend(self._field_rows((choice.field,))) + widgets.extend(self._field_rows(choice.fields)) await detail.mount(*widgets) # -- save + init (unchanged behaviour) --------------------------------- def _save_and_init(self) -> None: self._capture() - error = onboarding.validate_all(self._values, self._chosen_provider) + error = onboarding.validate_all(self._values, self._chosen) if error: self.notify(error, severity="error") return - non_empty = onboarding.collected_secrets(self._values, self._chosen_provider) + non_empty = onboarding.collected_secrets(self._values, self._chosen) save_env_secrets(self.app.root, non_empty) self.query_one("#onboard-go", Button).disabled = True self.run_worker(self._run_init(), exclusive=True) diff --git a/mcp_server/tui/config.py b/mcp_server/tui/config.py index 3881026..e9fbe15 100644 --- a/mcp_server/tui/config.py +++ b/mcp_server/tui/config.py @@ -33,6 +33,8 @@ "GITHUB_TOKEN", "GIT_USER_NAME", "GITHUB_ORG", + "BITBUCKET_TOKEN", + "BITBUCKET_WORKSPACE", "P10Y_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", @@ -43,6 +45,7 @@ MASKED_KEYS: frozenset[str] = frozenset( { "GITHUB_TOKEN", + "BITBUCKET_TOKEN", "P10Y_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", diff --git a/mcp_server/tui/onboarding.py b/mcp_server/tui/onboarding.py index 93cbb8a..12d6c68 100644 --- a/mcp_server/tui/onboarding.py +++ b/mcp_server/tui/onboarding.py @@ -10,10 +10,16 @@ (``ENV_SECRET_KEYS`` / ``MASKED_KEYS``) — never restated — and a module-load assertion fails loudly if a step ever references a key that isn't a known secret, so this content cannot silently drift from the writer in ``config.py``. + +A wizard can have more than one ``CHOICE`` step (LLM provider, git host, ...). +Each is independent: the "chosen option per step" state is a +``dict[step_id, option_id]`` (see ``default_choices``), and every function +below that used to take a single "chosen provider" string now takes that dict. """ from __future__ import annotations +import itertools from dataclasses import dataclass from enum import Enum @@ -23,18 +29,17 @@ PROVIDER_OPENROUTER = "openrouter" PROVIDER_ANTHROPIC = "anthropic" -# Secret keys for the two LLM providers (exactly one is collected per run). -PROVIDER_KEYS: dict[str, str] = { - PROVIDER_OPENROUTER: "OPENROUTER_API_KEY", - PROVIDER_ANTHROPIC: "ANTHROPIC_API_KEY", -} +# Provider option ids for the git-host choice step. A deployment is either +# all-GitHub or all-BitBucket, never mixed — mirrors the backend's GitProvider. +GIT_PROVIDER_GITHUB = "github" +GIT_PROVIDER_BITBUCKET = "bitbucket_cloud" class StepKind(Enum): """What a wizard step renders/collects.""" INFO = "info" # explanatory only — no fields - CHOICE = "choice" # provider pick — reveals one field for the choice + CHOICE = "choice" # pick one option — reveals that option's field(s) FIELDS = "fields" # one or more plain input rows REVIEW = "review" # read-only recap + Save & Initialize @@ -56,14 +61,14 @@ def masked(self) -> bool: @dataclass(frozen=True) class Choice: - """One option in a ``CHOICE`` step, revealing a single field when picked.""" + """One option in a ``CHOICE`` step, revealing its field(s) when picked.""" option_id: str label: str why: str how_to: tuple[str, ...] url: str - field: Field + fields: tuple[Field, ...] @dataclass(frozen=True) @@ -123,7 +128,7 @@ class Step: "2. Create a key and paste it below.", ), url="https://openrouter.ai/keys", - field=Field("OPENROUTER_API_KEY", "OpenRouter API key", required=True), + fields=(Field("OPENROUTER_API_KEY", "OpenRouter API key", required=True),), ), Choice( option_id=PROVIDER_ANTHROPIC, @@ -134,41 +139,76 @@ class Step: "2. Create a key and paste it below.", ), url="https://console.anthropic.com/settings/keys", - field=Field("ANTHROPIC_API_KEY", "Anthropic API key", required=True), + fields=(Field("ANTHROPIC_API_KEY", "Anthropic API key", required=True),), ), ), ) -_GITHUB = Step( - step_id="github", - title="GitHub access", - kind=StepKind.FIELDS, +_GIT = Step( + step_id="git", + title="Choose your git host", + kind=StepKind.CHOICE, why=( - "A GitHub Personal Access Token lets SpecFlow create and manage the " - "disposable workspace repos where agents commit generated code. Create " - "ONE token here and reuse it for the Compass integration in the next " - "step — no second GitHub key needed." + "SpecFlow needs exactly one git host to create and manage the disposable " + "workspace repos where agents commit generated code. A deployment is " + "either all-GitHub or all-BitBucket, never mixed." ), - how_to=( - "1. Open https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", - "2. Create a classic PAT. SpecFlow always uses `repo` + `read:user` (to create", - " workspace repos and resolve your GitHub login), plus `workflow` for deploy/E2E", - " runs; `admin:repo_hook` + full `user` are added so the SAME token also works", - " for the Compass GitHub integration in the next step.", - "3. Paste it below.", - ), - url="https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", - fields=( - Field("GITHUB_TOKEN", "GitHub token", required=True), - Field( - "GITHUB_ORG", - "GitHub org (optional)", - hint="GH org for workspace repos; blank = your GitHub login", + default_choice=GIT_PROVIDER_GITHUB, + choices=( + Choice( + option_id=GIT_PROVIDER_GITHUB, + label="GitHub (default)", + why=( + "A GitHub Personal Access Token lets SpecFlow create and manage the " + "disposable workspace repos. Create ONE token here and reuse it for " + "the Compass integration in the next step — no second GitHub key needed." + ), + how_to=( + "1. Open https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", + "2. Create a classic PAT. SpecFlow always uses `repo` + `read:user` (to create", + " workspace repos and resolve your GitHub login), plus `workflow` for deploy/E2E", + " runs; `admin:repo_hook` + full `user` are added so the SAME token also works", + " for the Compass GitHub integration in the next step.", + "3. Paste it below.", + ), + url="https://github.com/settings/tokens/new?scopes=repo,read:user,workflow,admin:repo_hook,user", + fields=( + Field("GITHUB_TOKEN", "GitHub token", required=True), + Field( + "GITHUB_ORG", + "GitHub org (optional)", + hint="GH org for workspace repos; blank = your GitHub login", + ), + Field( + "GIT_USER_NAME", + "GitHub username (optional)", + hint="auto-resolved from your GitHub login (GET /user) when blank", + ), + ), ), - Field( - "GIT_USER_NAME", - "GitHub username (optional)", - hint="auto-resolved from your GitHub login (GET /user) when blank", + Choice( + option_id=GIT_PROVIDER_BITBUCKET, + label="BitBucket Cloud", + why=( + "A BitBucket Cloud Workspace access token lets SpecFlow create and " + "manage the disposable workspace repos. No username needed — access " + "tokens always authenticate as the fixed actor `x-token-auth`." + ), + how_to=( + "1. Open your BitBucket workspace > Settings > Access tokens > Create access token.", + "2. Grant it repository read, write, and admin scopes.", + "3. Paste the token and your workspace slug below.", + ), + url="https://bitbucket.org", + fields=( + Field("BITBUCKET_TOKEN", "BitBucket access token", required=True), + Field( + "BITBUCKET_WORKSPACE", + "BitBucket workspace", + required=True, + hint="workspace slug that owns the workspace repos", + ), + ), ), ), ) @@ -184,10 +224,9 @@ class Step: ), how_to=( "1. Create a Compass account at https://compass.p10y.com (enterprise required).", - "2. Connect your GitHub org: Settings > Integrations > New Integration > GitHub;", - " add the GH username/org owning the workspace repos; paste the SAME GitHub", - " PAT you created in the previous step (it already has the scopes Compass", - " needs: admin:repo_hook, repo:*, user:*, workflow:*); enable auto-discovery; save.", + "2. Connect your git host: Settings > Integrations > New Integration;", + " add the username/org/workspace owning the workspace repos; paste the SAME", + " token you created in the previous step; enable auto-discovery; save.", "3. Generate an API token: Settings > API Tokens > Generate.", "Docs: docs/quickstart-compass.md. init auto-resolves P10Y_ORGANISATION_ID.", ), @@ -205,10 +244,12 @@ class Step: ), ) -STEPS: tuple[Step, ...] = (_WELCOME, _PROVIDER, _GITHUB, _COMPASS, _REVIEW) +STEPS: tuple[Step, ...] = (_WELCOME, _PROVIDER, _GIT, _COMPASS, _REVIEW) -# The provider-choice step, exposed so the screen need not hardcode an index. +# Choice steps, exposed so the screen need not hardcode which steps are CHOICE. PROVIDER_STEP: Step = _PROVIDER +GIT_STEP: Step = _GIT +CHOICE_STEPS: tuple[Step, ...] = tuple(s for s in STEPS if s.kind is StepKind.CHOICE) # --------------------------------------------------------------------------- @@ -221,7 +262,8 @@ def _all_field_keys() -> set[str]: keys: set[str] = set() for step in STEPS: keys.update(f.key for f in step.fields) - keys.update(c.field.key for c in step.choices) + for choice in step.choices: + keys.update(f.key for f in choice.fields) return keys @@ -233,29 +275,52 @@ def _all_field_keys() -> set[str]: # --------------------------------------------------------------------------- # Pure validation + collection +# +# ``chosen`` is a dict[step_id, option_id] — one entry per CHOICE step. Use +# ``default_choices()`` to seed it. # --------------------------------------------------------------------------- -def provider_field(chosen_provider: str) -> Field: - """The single LLM key field for the chosen provider.""" - choice = next(c for c in _PROVIDER.choices if c.option_id == chosen_provider) - return choice.field +def default_choices() -> dict[str, str]: + """The default option id for every CHOICE step, keyed by step_id.""" + return {step.step_id: step.default_choice for step in CHOICE_STEPS} + + +def choice_fields(step: Step, chosen_option: str) -> tuple[Field, ...]: + """The field(s) revealed by the chosen option of a CHOICE step.""" + choice = next(c for c in step.choices if c.option_id == chosen_option) + return choice.fields + + +def _field_by_key() -> dict[str, Field]: + field_by_key: dict[str, Field] = {f.key: f for step in STEPS for f in step.fields} + for step in CHOICE_STEPS: + for choice in step.choices: + field_by_key.update({f.key: f for f in choice.fields}) + return field_by_key -def required_keys(chosen_provider: str) -> list[str]: +def required_keys(chosen: dict[str, str]) -> list[str]: """All keys that must be non-empty for a complete setup, in step order.""" - keys: list[str] = [provider_field(chosen_provider).key] + keys: list[str] = [] for step in STEPS: - keys.extend(f.key for f in step.fields if f.required) + if step.kind is StepKind.CHOICE: + keys.extend(f.key for f in choice_fields(step, chosen[step.step_id]) if f.required) + else: + keys.extend(f.key for f in step.fields if f.required) return keys -def validate_step(step: Step, values: dict[str, str], chosen_provider: str) -> str | None: +def validate_step(step: Step, values: dict[str, str], chosen: dict[str, str]) -> str | None: """Return an error string if the step's required inputs are unmet, else None.""" if step.kind is StepKind.CHOICE: - f = provider_field(chosen_provider) - if not values.get(f.key, "").strip(): - return f"{f.label} is required — paste it to continue." + missing = [ + f.label + for f in choice_fields(step, chosen[step.step_id]) + if f.required and not values.get(f.key, "").strip() + ] + if missing: + return "Missing required field(s): " + ", ".join(missing) return None if step.kind is StepKind.FIELDS: missing = [f.label for f in step.fields if f.required and not values.get(f.key, "").strip()] @@ -264,17 +329,12 @@ def validate_step(step: Step, values: dict[str, str], chosen_provider: str) -> s return None -def validate_all(values: dict[str, str], chosen_provider: str) -> str | None: - """Final gate before save — every required key must be present. - - Reproduces the original ``_validation_error`` rule (GitHub token + P10Y key - + exactly the chosen LLM provider key) so the contract lives in one place. - """ - field_by_key = {f.key: f for step in STEPS for f in step.fields} - field_by_key.update({c.field.key: c.field for c in _PROVIDER.choices}) +def validate_all(values: dict[str, str], chosen: dict[str, str]) -> str | None: + """Final gate before save — every required key (for the chosen options) must be present.""" + field_by_key = _field_by_key() missing = [ field_by_key[key].label - for key in required_keys(chosen_provider) + for key in required_keys(chosen) if not values.get(key, "").strip() ] if missing: @@ -283,18 +343,28 @@ def validate_all(values: dict[str, str], chosen_provider: str) -> str | None: def env_satisfies_requirements(secrets: dict[str, str]) -> bool: - """True if an existing ``.env`` already has every required key for a provider. + """True if an existing ``.env`` already has every required key for some combination + of choices (e.g. some LLM provider AND some git host). - Reuses ``validate_all`` against each known provider so the "what's required" - contract stays defined in exactly one place (no second validator). A ``.env`` - is sufficient when it satisfies either provider's requirements. + Reuses ``validate_all`` against every combination so the "what's required" + contract stays defined in exactly one place (no second validator). """ - return any(validate_all(secrets, provider) is None for provider in PROVIDER_KEYS) - - -def collected_secrets(values: dict[str, str], chosen_provider: str) -> dict[str, str]: - """Non-empty values to write to ``.env``, omitting the non-chosen provider key.""" - drop = {k for k in PROVIDER_KEYS.values() if k != provider_field(chosen_provider).key} + option_lists = [[c.option_id for c in step.choices] for step in CHOICE_STEPS] + for combo in itertools.product(*option_lists): + chosen = {step.step_id: option for step, option in zip(CHOICE_STEPS, combo)} + if validate_all(secrets, chosen) is None: + return True + return False + + +def collected_secrets(values: dict[str, str], chosen: dict[str, str]) -> dict[str, str]: + """Non-empty values to write to ``.env``, omitting fields for unchosen options.""" + drop: set[str] = set() + for step in CHOICE_STEPS: + chosen_option = chosen[step.step_id] + for choice in step.choices: + if choice.option_id != chosen_option: + drop.update(f.key for f in choice.fields) return { key: value.strip() for key, value in values.items()