From aa6fa7b19c1fa30d525f20a2b90643fa145d0cc9 Mon Sep 17 00:00:00 2001 From: jc Date: Thu, 1 Jan 2026 14:50:07 -0500 Subject: [PATCH 1/5] feat: comprehensive CLI refactoring and infrastructure improvements Major Changes: - Refactored CLI to use dependency injection pattern with CLIContext - Consolidated database workflows into shared package eliminating ~600 lines of duplication - Split monolithic entity command (555 lines) into modular package - Added ComposeRunner helper for consistent docker-compose operations - Consolidated deployer base with restart_container method - Introduced DbRuntime protocol with Compose and K8s adapters Database & Infrastructure: - Fixed Redis authentication issue in development environment - Fixed test_07 container detection bug with exact regex matching - Removed redundant get_settings() call in run_sync workflow - Added PostgreSQL TLS verification job for Helm deployments - Improved secret management flow and documentation Testing: - Added comprehensive workflow integration tests (test_db_runtime.py, test_db_workflows.py) - Improved test coverage for CLI context and database workflows - Fixed Loguru logging errors during test runs - All tests passing (15/15 in db_runtime, 15/15 in db_workflows) Documentation: - Updated security documentation for OIDC hardening - Documented refresh token policy and JWKS cache controls - Added secrets management flow documentation - Improved deployment and Helm migration documentation Code Quality: - Fixed type annotations (runtime_k8s.py) - Removed dead code (helpers.py, cleanup scripts) - Improved imports and code organization - Maintained strict mypy compliance Files Changed: 132 files, +12091 insertions, -2140 deletions --- .env.example | 5 +- README.md | 9 +- alembic.ini | 54 ++ config.yaml | 170 ++-- docker-compose.prod.yml | 57 +- docs/cli-infra-reorg-plan.md | 92 ++ docs/database-migrations.md | 564 +++++++++++ docs/fastapi-kubernetes-deployment.md | 70 ++ ...pi-production-deployment-docker-compose.md | 7 + docs/index.md | 21 +- docs/infra/PRODUCTION_DEPLOYMENT.md | 41 +- infra/docker/dev/cleanup_dev.sh | 28 +- infra/docker/dev/keycloak/setup_script.py | 4 +- infra/docker/dev/setup_dev.sh | 16 +- .../postgres/admin-scripts/verify-init.sh | 99 -- .../prod/temporal/scripts/entrypoint.sh | 46 + .../prod/temporal/scripts/schema-setup.sh | 45 +- .../api-forge-bundled-postgres/Chart.yaml | 15 + .../api-forge-bundled-postgres/FEATURES.md | 277 ++++++ .../files/.gitignore | 10 + .../templates/configmap-env.yaml | 44 + .../templates/configmap.yaml} | 6 +- .../templates/networkpolicy.yaml | 31 + .../templates/poddisruptionbudget.yaml | 16 + .../templates/pvc-backups.yaml} | 8 +- .../templates/secrets/postgres-ca.yaml | 3 + .../templates/secrets/postgres-secrets.yaml | 3 + .../templates/secrets/postgres-tls.yaml | 3 + .../templates/service.yaml} | 4 +- .../templates/statefulset.yaml} | 94 +- .../api-forge-bundled-postgres/values.yaml | 81 ++ .../templates/configmaps/app-env.yaml | 4 +- .../templates/jobs/postgres-verifier.yaml | 12 +- .../jobs/temporal-namespace-init.yaml | 14 +- .../templates/jobs/temporal-schema-setup.yaml | 11 +- .../network-policies/postgres-netpol.yaml | 17 - .../poddisruptionbudgets/postgres-pdb.yaml | 23 - .../templates/secrets/postgres-ca.yaml | 4 + .../templates/secrets/postgres-secrets.yaml | 4 + .../templates/storage/postgres-data-pvc.yaml | 20 - .../api-forge/values-bitnami-postgres.yaml | 54 ++ infra/helm/api-forge/values.yaml | 72 +- infra/scripts/entrypoint.sh | 29 - infra/secrets/generate_secrets.sh | 10 + migrations/README.md | 229 +++++ migrations/env.py | 98 ++ migrations/script.py.mako | 25 + .../versions/20251224_0327_initial_schema.py | 112 +++ pyproject.toml | 4 + src/app/core/services/database/db_manage.py | 4 +- src/app/entities/__init__.py | 8 + src/app/entities/loader.py | 72 ++ src/app/runtime/config/config_data.py | 302 +++--- src/app/runtime/config/config_loader.py | 4 +- src/app/runtime/config/config_utils.py | 43 +- src/cli/__init__.py | 13 +- src/cli/commands/__init__.py | 4 +- src/cli/commands/db/__init__.py | 28 + src/cli/commands/db/runtime.py | 32 + src/cli/commands/db/runtime_compose.py | 32 + src/cli/commands/db/runtime_k8s.py | 40 + src/cli/commands/db/workflows.py | 249 +++++ src/cli/commands/db_utils.py | 417 +++++++++ src/cli/commands/dev.py | 61 +- src/cli/commands/entity/__init__.py | 5 + src/cli/commands/{entity.py => entity/cli.py} | 251 +---- src/cli/commands/entity/scaffold.py | 198 ++++ src/cli/commands/entity/templates.py | 26 + src/cli/commands/fly.py | 10 +- src/cli/commands/k8s.py | 186 ++-- src/cli/commands/k8s_db.py | 884 ++++++++++++++++++ src/cli/commands/prod.py | 212 +++-- src/cli/commands/prod_db.py | 758 +++++++++++++++ src/cli/commands/secrets.py | 51 +- src/cli/commands/shared.py | 143 --- src/cli/commands/users.py | 5 +- src/cli/context.py | 52 ++ src/cli/deployment/base.py | 90 +- src/cli/deployment/constants.py | 13 + src/cli/deployment/dev_deployer.py | 43 +- src/cli/deployment/helm_deployer/__init__.py | 2 - src/cli/deployment/helm_deployer/cleanup.py | 60 +- .../deployment/helm_deployer/config_sync.py | 47 +- src/cli/deployment/helm_deployer/deployer.py | 356 ++++++- .../deployment/helm_deployer/helm_release.py | 144 ++- .../deployment/helm_deployer/image_builder.py | 97 +- .../helm_deployer/secret_manager.py | 6 +- src/cli/deployment/helm_deployer/validator.py | 113 ++- src/cli/deployment/prod_deployer.py | 333 ++++--- src/cli/deployment/shell_commands/__init__.py | 66 +- src/cli/deployment/shell_commands/docker.py | 12 +- src/cli/deployment/shell_commands/kubectl.py | 220 ----- src/cli/deployment/status_display.py | 57 +- src/cli/shared/__init__.py | 0 src/cli/shared/compose.py | 79 ++ src/cli/shared/console.py | 267 ++++++ src/cli/shared/secrets.py | 29 + .../helm_deployer => infra}/constants.py | 45 +- src/infra/docker_compose/__init__.py | 0 .../docker_compose/postgres_connection.py | 74 ++ src/infra/k8s/__init__.py | 18 + src/infra/k8s/controller.py | 370 +++++++- src/infra/k8s/controller.pyi | 288 ++++++ src/infra/k8s/helpers.py | 43 + src/infra/k8s/kr8s_controller.py | 75 +- src/infra/k8s/kubectl_controller.py | 54 +- src/infra/k8s/port_forward.py | 478 ++++++++++ src/infra/k8s/postgres_connection.py | 116 +++ src/infra/k8s/utils.py | 23 +- src/infra/postgres/__init__.py | 26 + src/infra/postgres/backup.py | 199 ++++ src/infra/postgres/connection.py | 467 +++++++++ src/infra/postgres/init.py | 278 ++++++ src/infra/postgres/migrations.py | 170 ++++ src/infra/postgres/reset.py | 179 ++++ src/infra/postgres/sync.py | 179 ++++ src/infra/postgres/verify.py | 416 +++++++++ .../utils}/service_config.py | 39 +- src/utils/console_like.py | 43 + src/utils/paths.py | 23 + tests/conftest.py | 7 +- tests/e2e/test_copier_to_deployment.py | 241 ++++- .../app/core/config/test_database_config.py | 110 ++- .../unit/cli/deployment/test_image_builder.py | 41 +- tests/unit/cli/deployment/test_validator.py | 141 ++- tests/unit/cli/test_cli_context.py | 173 ++++ tests/unit/cli/test_compose_runner.py | 290 ++++++ tests/unit/cli/test_console_error_handling.py | 27 + tests/unit/cli/test_db_runtime.py | 354 +++++++ tests/unit/cli/test_db_utils.py | 65 ++ tests/unit/cli/test_db_workflows.py | 349 +++++++ uv.lock | 41 + 132 files changed, 12091 insertions(+), 2140 deletions(-) create mode 100644 alembic.ini create mode 100644 docs/cli-infra-reorg-plan.md create mode 100644 docs/database-migrations.md delete mode 100644 infra/docker/prod/postgres/admin-scripts/verify-init.sh create mode 100644 infra/helm/api-forge-bundled-postgres/Chart.yaml create mode 100644 infra/helm/api-forge-bundled-postgres/FEATURES.md create mode 100644 infra/helm/api-forge-bundled-postgres/files/.gitignore create mode 100644 infra/helm/api-forge-bundled-postgres/templates/configmap-env.yaml rename infra/helm/{api-forge/templates/configmaps/postgres-config.yaml => api-forge-bundled-postgres/templates/configmap.yaml} (79%) create mode 100644 infra/helm/api-forge-bundled-postgres/templates/networkpolicy.yaml create mode 100644 infra/helm/api-forge-bundled-postgres/templates/poddisruptionbudget.yaml rename infra/helm/{api-forge/templates/storage/postgres-backups-pvc.yaml => api-forge-bundled-postgres/templates/pvc-backups.yaml} (64%) create mode 100644 infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-ca.yaml create mode 100644 infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-secrets.yaml create mode 100644 infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-tls.yaml rename infra/helm/{api-forge/templates/services/postgres-service.yaml => api-forge-bundled-postgres/templates/service.yaml} (81%) rename infra/helm/{api-forge/templates/deployments/postgres.yaml => api-forge-bundled-postgres/templates/statefulset.yaml} (66%) create mode 100644 infra/helm/api-forge-bundled-postgres/values.yaml delete mode 100644 infra/helm/api-forge/templates/network-policies/postgres-netpol.yaml delete mode 100644 infra/helm/api-forge/templates/poddisruptionbudgets/postgres-pdb.yaml delete mode 100644 infra/helm/api-forge/templates/storage/postgres-data-pvc.yaml create mode 100644 infra/helm/api-forge/values-bitnami-postgres.yaml delete mode 100644 infra/scripts/entrypoint.sh create mode 100644 migrations/README.md create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/20251224_0327_initial_schema.py create mode 100644 src/app/entities/loader.py create mode 100644 src/cli/commands/db/__init__.py create mode 100644 src/cli/commands/db/runtime.py create mode 100644 src/cli/commands/db/runtime_compose.py create mode 100644 src/cli/commands/db/runtime_k8s.py create mode 100644 src/cli/commands/db/workflows.py create mode 100644 src/cli/commands/db_utils.py create mode 100644 src/cli/commands/entity/__init__.py rename src/cli/commands/{entity.py => entity/cli.py} (55%) create mode 100644 src/cli/commands/entity/scaffold.py create mode 100644 src/cli/commands/entity/templates.py create mode 100644 src/cli/commands/k8s_db.py create mode 100644 src/cli/commands/prod_db.py delete mode 100644 src/cli/commands/shared.py create mode 100644 src/cli/context.py create mode 100644 src/cli/deployment/constants.py delete mode 100644 src/cli/deployment/shell_commands/kubectl.py create mode 100644 src/cli/shared/__init__.py create mode 100644 src/cli/shared/compose.py create mode 100644 src/cli/shared/console.py create mode 100644 src/cli/shared/secrets.py rename src/{cli/deployment/helm_deployer => infra}/constants.py (70%) create mode 100644 src/infra/docker_compose/__init__.py create mode 100644 src/infra/docker_compose/postgres_connection.py create mode 100644 src/infra/k8s/controller.pyi create mode 100644 src/infra/k8s/helpers.py create mode 100644 src/infra/k8s/port_forward.py create mode 100644 src/infra/k8s/postgres_connection.py create mode 100644 src/infra/postgres/__init__.py create mode 100644 src/infra/postgres/backup.py create mode 100644 src/infra/postgres/connection.py create mode 100644 src/infra/postgres/init.py create mode 100644 src/infra/postgres/migrations.py create mode 100644 src/infra/postgres/reset.py create mode 100644 src/infra/postgres/sync.py create mode 100644 src/infra/postgres/verify.py rename src/{cli/deployment => infra/utils}/service_config.py (65%) create mode 100644 src/utils/console_like.py create mode 100644 tests/unit/cli/test_cli_context.py create mode 100644 tests/unit/cli/test_compose_runner.py create mode 100644 tests/unit/cli/test_console_error_handling.py create mode 100644 tests/unit/cli/test_db_runtime.py create mode 100644 tests/unit/cli/test_db_utils.py create mode 100644 tests/unit/cli/test_db_workflows.py diff --git a/.env.example b/.env.example index f67c982..4a0a8b5 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,11 @@ PRODUCTION_BASE_URL=http://localhost:8000 DEVELOPMENT_BASE_URL=http://localhost:8000 ### ----------------- Database Settings ----------------- ### -PRODUCTION_DATABASE_URL=postgresql://appuser@postgres:5432/appdb?sslmode=require +PRODUCTION_DATABASE_URL=postgresql://postgres:5432?sslmode=require DEVELOPMENT_DATABASE_URL=postgresql://appuser:devpass@localhost:5433/appdb +PG_SUPERUSER=postgres +PG_DB=postgres APP_DB=appdb APP_DB_OWNER=appowner APP_DB_USER=appuser @@ -27,6 +29,7 @@ DEVELOPMENT_TEMPORAL_URL=localhost:7234 PRODUCTION_TEMPORAL_URL=temporal:7233 TEMPORAL_DB_USER=temporaluser +TEMPORAL_DB_OWNER=temporalowner TEMPORAL_DB=temporal TEMPORAL_VIS_DB=temporal_visibility ### ----------------- OIDC Configuration ----------------- ### diff --git a/README.md b/README.md index 20a83ff..81c69ee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Build your next SaaS backend, internal API gateway, or microservice with a pre-c * **FastAPI** – high-performance Python web framework * **SQLAlchemy** and **SQLModel** – ORM and typed models for data persistence * **Pydantic** – data validation and type safety -* **PostgreSQL** – production-ready relational database +* **PostgreSQL** – production-ready relational database (bundled or external) * **Redis** – caching, sessions, and rate limiting * **Temporal** – background workflows and reliable task orchestration * **Docker** – containerized development and deployment @@ -195,6 +195,13 @@ api-forge-cli deploy down prod --volumes api-forge-cli deploy up k8s api-forge-cli deploy status k8s api-forge-cli deploy down k8s + +# Database management (Kubernetes) +api-forge-cli k8s db init # Initialize database with roles/schema +api-forge-cli k8s db verify # Verify database setup and credentials +api-forge-cli k8s db sync # Sync local password files to database +api-forge-cli k8s db status # Show database health metrics +api-forge-cli k8s db backup # Create database backup ``` ### Entity scaffolding diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..2b2774c --- /dev/null +++ b/alembic.ini @@ -0,0 +1,54 @@ +# Alembic configuration file for database migrations + +[alembic] +# Path to migration scripts +script_location = migrations + +# Template used to generate migration files +file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(slug)s + +# Timezone for migration timestamps +timezone = UTC + +# Truncate slug to 40 characters +truncate_slug_length = 40 + +# Revision environment configuration +# This is used by env.py to configure the migration environment +[alembic:env] +# sqlalchemy.url is set dynamically in env.py from DATABASE_URL + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/config.yaml b/config.yaml index a5da3db..67df37d 100644 --- a/config.yaml +++ b/config.yaml @@ -5,33 +5,40 @@ config: enabled: true per_endpoint: true per_method: true - database: url: "${DATABASE_URL:-postgresql+asyncpg://appuser@postgres:5432/appdb}" + pg_superuser: "${PG_SUPERUSER:-postgres}" + pg_db: "${PG_DB:-postgres}" app_db: "${APP_DB:-appdb}" owner_user: "${APP_DB_OWNER:-appowner}" user: "${APP_DB_USER:-appuser}" ro_user: "${APP_DB_RO_USER:-backupuser}" + temporal_user: "${TEMPORAL_DB_USER:-temporaluser}" + temporal_owner: "${TEMPORAL_DB_OWNER:-temporalowner}" pool_size: 20 max_overflow: 10 pool_timeout: 30 pool_recycle: 1800 environment_mode: "${APP_ENVIRONMENT:-development}" - password_file_path: "${DATABASE_PASSWORD_FILE_PATH:-/run/secrets/postgres_app_user_pw}" - password_env_var: "${DATABASE_PASSWORD_ENV_VAR:-POSTGRES_APP_USER_PW}" + bundled_postgres: + enabled: true + password_file_path: "${DATABASE_PASSWORD_FILE_PATH:-/run/secrets/postgres_app_user_pw}" + password_env_var: "${DATABASE_PASSWORD_ENV_VAR:-POSTGRES_APP_USER_PW}" temporal: enabled: true url: "${TEMPORAL_URL:-temporal:7233}" - namespace: "default" - task_queue: "default" + db_user: "${TEMPORAL_DB_USER:-temporaluser}" + db_owner: "${TEMPORAL_DB_OWNER:-temporalowner}" + namespace: default + task_queue: default workflows: - execution_timeout_s: 86400 # 24 hours. Maximum time a workflow execution can run. Includes retries and continue as new. - run_timeout_s: 7200 # 2 hours. Maximum time for a single run of a workflow (before retries). - task_timeout_s: 10 # 10 seconds. Maximum time to complete a workflow task. + execution_timeout_s: 86400 + run_timeout_s: 7200 + task_timeout_s: 10 activities: - start_to_close_timeout_s: 1200 # 20 minutes. Limits the maximum execution time of a single execution of an activity. - schedule_to_close_timeout_s: 3600 # 1 hour. Limits the total time allotted for an activity from scheduling to completion. Includes retries. - heartbeat_timeout_s: 300 # 5 minutes. If an activity does not report progress within this time, it is considered failed. + start_to_close_timeout_s: 1200 + schedule_to_close_timeout_s: 3600 + heartbeat_timeout_s: 300 retry: maximum_attempts: 5 initial_interval_seconds: 5 @@ -47,11 +54,13 @@ config: max_workflow_tasks_per_second: 100 max_concurrent_workflow_tasks: 100 sticky_queue_schedule_to_start_timeout_ms: 10000 - worker_build_id: "api-worker-1" + worker_build_id: api-worker-1 redis: enabled: true url: "${REDIS_URL:-redis://localhost:6379}" password: "${REDIS_PASSWORD:-}" + password_file_path: "${REDIS_PASSWORD_FILE_PATH:-/app/keys/redis_password}" + password_env_var: "${REDIS_PASSWORD_ENV_VAR:-REDIS_PASSWORD}" max_connections: 10 decode_responses: true socket_timeout: 5 @@ -61,53 +70,53 @@ config: google: enabled: true dev_only: false - authorization_endpoint: "https://accounts.google.com/o/oauth2/v2/auth" - token_endpoint: "https://oauth2.googleapis.com/token" - userinfo_endpoint: "https://openidconnect.googleapis.com/v1/userinfo" - end_session_endpoint: "https://accounts.google.com/logout" - issuer: "https://accounts.google.com" - jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" + authorization_endpoint: https://accounts.google.com/o/oauth2/v2/auth + token_endpoint: https://oauth2.googleapis.com/token + userinfo_endpoint: https://openidconnect.googleapis.com/v1/userinfo + end_session_endpoint: https://accounts.google.com/logout + issuer: https://accounts.google.com + jwks_uri: https://www.googleapis.com/oauth2/v3/certs scopes: - - openid - - profile - - email + - openid + - profile + - email client_id: your-google-client-id client_secret: "${OIDC_GOOGLE_CLIENT_SECRET}" redirect_uri: "${OIDC_GOOGLE_REDIRECT_URI:-http://localhost:8000/auth/google/callback}" microsoft: enabled: true dev_only: false - authorization_endpoint: "https://login.microsoftonline.com/common/oauth2/v2.0/authorize" - token_endpoint: "https://login.microsoftonline.com/common/oauth2/v2.0/token" - userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo" - end_session_endpoint: "https://login.microsoftonline.com/common/oauth2/v2.0/logout" - issuer: "https://login.microsoftonline.com" - jwks_uri: "https://login.microsoftonline.com/common/discovery/v2.0/keys" + authorization_endpoint: https://login.microsoftonline.com/common/oauth2/v2.0/authorize + token_endpoint: https://login.microsoftonline.com/common/oauth2/v2.0/token + userinfo_endpoint: https://graph.microsoft.com/oidc/userinfo + end_session_endpoint: https://login.microsoftonline.com/common/oauth2/v2.0/logout + issuer: https://login.microsoftonline.com + jwks_uri: https://login.microsoftonline.com/common/discovery/v2.0/keys scopes: - - openid - - profile - - email + - openid + - profile + - email client_id: your-microsoft-client-id client_secret: "${OIDC_MICROSOFT_CLIENT_SECRET}" redirect_uri: "${OIDC_MICROSOFT_REDIRECT_URI:-http://localhost:8000/auth/microsoft/callback}" keycloak: enabled: true dev_only: true - issuer: "http://localhost:8080/realms/test-realm" + issuer: http://localhost:8080/realms/test-realm openid_configuration_endpoint: "${OIDC_KEYCLOAK_ISSUER:-http://localhost:8080/realms/test-realm}/.well-known/openid-configuration" client_id: test-client client_secret: test-client-secret scopes: - - openid - - profile - - email + - openid + - profile + - email redirect_uri: "${OIDC_KEYCLOAK_REDIRECT_URI:-http://localhost:8000/auth/web/callback}" jwks_uri: "${OIDC_KEYCLOAK_JWKS_URI:-http://localhost:8080/realms/test-realm/protocol/openid-connect/certs}" end_session_endpoint: "${OIDC_KEYCLOAK_END_SESSION_ENDPOINT:-http://localhost:8080/realms/test-realm/protocol/openid-connect/logout}" userinfo_endpoint: "${OIDC_KEYCLOAK_USERINFO_ENDPOINT:-http://localhost:8080/realms/test-realm/protocol/openid-connect/userinfo}" authorization_endpoint: "${OIDC_KEYCLOAK_AUTHORIZATION_ENDPOINT:-http://localhost:8080/realms/test-realm/protocol/openid-connect/auth}" token_endpoint: "${OIDC_KEYCLOAK_TOKEN_ENDPOINT:-http://localhost:8080/realms/test-realm/protocol/openid-connect/token}" - default_provider: "keycloak" + default_provider: keycloak global_redirect_uri: "${OIDC_REDIRECT_URI:-http://localhost:8000/auth/callback}" allowed_redirect_hosts: [] allowed_audiences: [] @@ -116,74 +125,67 @@ config: persist_in_session_store: false max_session_lifetime_seconds: 86400 jwt: - # JWT Validation Settings - allowed_algorithms: - - "RS256" - - "RS512" - - "ES256" - - "ES384" - - "HS256" # Only if you have shared secrets - # Audiences that your API accepts (who the tokens are intended for) - gen_issuer: "${BASE_URL:-my-api-issuer}" # Issuer name to use when generating tokens + allowed_algorithms: + - RS256 + - RS512 + - ES256 + - ES384 + - HS256 + gen_issuer: "${BASE_URL:-my-api-issuer}" audiences: - - "${JWT_AUDIENCE:-api://default}" - - "${JWT_AUDIENCE_SECONDARY:-http://localhost:8000}" - # Clock skew tolerance in seconds (accounts for time differences between servers) + - "${JWT_AUDIENCE:-api://default}" + - "${JWT_AUDIENCE_SECONDARY:-http://localhost:8000}" clock_skew: 60 - # Token validation settings verify_signature: true - verify_exp: true # Verify expiration - verify_nbf: true # Verify not-before - verify_iat: true # Verify issued-at + verify_exp: true + verify_nbf: true + verify_iat: true require_exp: true require_iat: true - # Claim mappings (how to extract user info from JWT tokens) claims: - user_id: "${JWT_CLAIM_USER_ID:-sub}" # Usually 'sub' (subject) - email: "${JWT_CLAIM_EMAIL:-email}" # Email claim - roles: "${JWT_CLAIM_ROLES:-roles}" # Roles/permissions - groups: "${JWT_CLAIM_GROUPS:-groups}" # User groups - scope: "${JWT_CLAIM_SCOPE:-scope}" # OAuth scopes - name: "${JWT_CLAIM_NAME:-name}" # User's full name + user_id: "${JWT_CLAIM_USER_ID:-sub}" + email: "${JWT_CLAIM_EMAIL:-email}" + roles: "${JWT_CLAIM_ROLES:-roles}" + groups: "${JWT_CLAIM_GROUPS:-groups}" + scope: "${JWT_CLAIM_SCOPE:-scope}" + name: "${JWT_CLAIM_NAME:-name}" preferred_username: "${JWT_CLAIM_USERNAME:-preferred_username}" jwks_cache_ttl_seconds: 3600 jwks_cache_max_entries: 16 logging: level: "${LOG_LEVEL:-DEBUG}" - format: "${LOG_FORMAT:-json}" # Options: "json", "plain" + format: "${LOG_FORMAT:-json}" file: "${LOG_FILE:-logs/app.log}" max_size_mb: 5 backup_count: 5 app: - environment: "${APP_ENVIRONMENT:-development}" # Options: "development", "testing", "production" + environment: "${APP_ENVIRONMENT:-development}" host: "${APP_HOST:-localhost}" port: "${APP_PORT:-8000}" - session_max_age: ${SESSION_MAX_AGE:-3600} # Session max age in seconds - session_signing_secret: "${SESSION_SIGNING_SECRET}" # Secret for signing session JWTs - csrf_signing_secret: "${CSRF_SIGNING_SECRET}" # Secret for signing CSRF tokens + session_max_age: "${SESSION_MAX_AGE:-3600}" + session_signing_secret: "${SESSION_SIGNING_SECRET}" + csrf_signing_secret: "${CSRF_SIGNING_SECRET}" cors: origins: - - "${CLIENT_ORIGIN:-http://localhost:3000}" + - "${CLIENT_ORIGIN:-http://localhost:3000}" allow_credentials: true allow_methods: - - GET - - POST - - PUT - - DELETE - - OPTIONS + - GET + - POST + - PUT + - DELETE + - OPTIONS allow_headers: - - Authorization - - Content-Type - - X-Requested-With - - Accept - - Origin - - User-Agent - - DNT - - Cache-Control - - X-Mx-ReqToken - - Keep-Alive - - X-Requested-With - - If-Modified-Since - - X-CSRF-Token - - + - Authorization + - Content-Type + - X-Requested-With + - Accept + - Origin + - User-Agent + - DNT + - Cache-Control + - X-Mx-ReqToken + - Keep-Alive + - X-Requested-With + - If-Modified-Since + - X-CSRF-Token diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2cf01e9..73ebd5c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -9,6 +9,7 @@ x-entrypoint-common-vars: &entrypoint-common-vars services: postgres: container_name: api-forge-postgres + profiles: ["postgres"] # Only started via db create command image: app_data_postgres_image build: context: ./infra/docker/prod @@ -58,38 +59,15 @@ services: - ./infra/docker/prod/postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro - ./infra/docker/prod/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$APP_DB_USER $${APP_DB:+-d $$APP_DB} -h 127.0.0.1"] + test: ["CMD-SHELL", "pg_isready -U postgres -h 127.0.0.1"] interval: 5s timeout: 5s retries: 20 networks: [backend] - postgres-verifier: - container_name: api-forge-postgres-verifier - image: postgres:15 - depends_on: - postgres: - condition: service_healthy - restart: no - env_file: - - .env - secrets: - - postgres_app_user_pw # for APP_USER auth - - postgres_password # ⬅️ superuser (postgres) password, used only for pg_hba checks - volumes: - - ./infra/docker/prod/postgres/verify-init.sh:/verify.sh:ro - entrypoint: /bin/sh - command: - - -ec - - | - set -euo pipefail - export PGPASSWORD="$(cat /run/secrets/postgres_app_user_pw)" # APP_USER for most checks - export PGSU_PASSWORD="$(cat /run/secrets/postgres_password)" # ⬅️ superuser just for pg_hba checks - exec /verify.sh - networks: - - backend + # NOTE: postgres-verifier has been replaced by the CLI `db verify` command. + # Run `uv run api-forge-cli db verify` to verify PostgreSQL setup. - # Redis Cache/Session Store redis: container_name: api-forge-redis @@ -126,9 +104,6 @@ services: container_name: api-forge-temporal-schema-setup image: temporalio/admin-tools:1.29.0-tctl-1.18.4-cli-1.4.2 user: "0:0" # Run as root to read Docker secrets and copy them - depends_on: - postgres: - condition: service_healthy restart: "no" networks: [backend] environment: @@ -142,13 +117,15 @@ services: # Temporal schema setup config TEMPORAL_DB: ${TEMPORAL_DB:-temporal} TEMPORAL_VIS_DB: ${TEMPORAL_VIS_DB:-temporal_visibility} - EP: postgres + # EP is parsed from PRODUCTION_DATABASE_URL if not set (supports bundled & external) + EP: ${PG_HOST:-} + PRODUCTION_DATABASE_URL: ${PRODUCTION_DATABASE_URL:-} TEMPORAL_DB_USER: ${TEMPORAL_DB_USER:-temporaluser} PW_FILE: /tmp/secrets/postgres_temporal_pw # Points to copied secret # TLS settings - adjust as needed (can be overridden via .env) TLS_ENABLE: ${TEMPORAL_TLS_ENABLE:-true} - SSL_MODE: ${TEMPORAL_SSL_MODE:-verify-ca} # or 'require' / 'verify-full' + SSL_MODE: ${TEMPORAL_SSL_MODE:-verify-ca} # verify-ca validates cert against CA bundle TLS_CA_FILE: /tmp/certs/ca-bundle.crt # Points to copied cert # Optional overrides: @@ -199,8 +176,6 @@ services: dockerfile: ./infra/docker/prod/temporal/Dockerfile image: my-temporal-server:latest depends_on: - postgres: - condition: service_healthy temporal-schema-setup: condition: service_completed_successfully environment: @@ -216,23 +191,25 @@ services: # Temporal server config SERVICES: "history,matching,worker,frontend" DB: 'postgres12' - POSTGRES_SEEDS: postgres + # POSTGRES_SEEDS parsed from PRODUCTION_DATABASE_URL by entrypoint if not set + POSTGRES_SEEDS: ${PG_HOST:-} + PRODUCTION_DATABASE_URL: ${PRODUCTION_DATABASE_URL:-} POSTGRES_USER: ${TEMPORAL_DB_USER:-temporaluser} POSTGRES_DB: ${TEMPORAL_DB:-temporal} # POSTGRES_PWD will be set by wrapper script from POSTGRES_TEMPORAL_PW SQL_TLS_ENABLED: "true" SQL_CA: /tmp/certs/ca-bundle.crt # Points to copied cert - SQL_HOST_VERIFICATION: "true" - SQL_HOST_NAME: postgres + SQL_HOST_VERIFICATION: ${TEMPORAL_SQL_HOST_VERIFICATION:-true} # Verify cert hostname matches + SQL_HOST_NAME: ${PG_HOST:-} # Parsed from PRODUCTION_DATABASE_URL by entrypoint BIND_ON_IP: "0.0.0.0" #Performance Tuning (optional) - NUM_HISTORY_SHARDS: 64 + NUM_HISTORY_SHARDS: 4 # Temporal DB settings DBNAME: ${TEMPORAL_DB:-temporal} VISIBILITY_DBNAME: ${TEMPORAL_VIS_DB:-temporal_visibility} - DB_PORT: 5432 + DB_PORT: ${PG_PORT:-5432} # Parsed from PRODUCTION_DATABASE_URL by entrypoint healthcheck: # Probes Temporal's gRPC health endpoint on the frontend (7233) test: ["CMD", "grpc_health_probe", "-addr=127.0.0.1:7233", "-service=temporal.api.workflowservice.v1.WorkflowService"] @@ -294,8 +271,6 @@ services: dockerfile: Dockerfile image: api-forge-app:latest depends_on: - postgres: - condition: service_healthy redis: condition: service_started temporal: @@ -382,8 +357,6 @@ services: dockerfile: Dockerfile image: api-forge-app:latest depends_on: - postgres: - condition: service_healthy redis: condition: service_started temporal: diff --git a/docs/cli-infra-reorg-plan.md b/docs/cli-infra-reorg-plan.md new file mode 100644 index 0000000..f4f4d61 --- /dev/null +++ b/docs/cli-infra-reorg-plan.md @@ -0,0 +1,92 @@ +# CLI/Infra Re-Org Plan + +## Goals +- Preserve behavior while improving readability, testability, and maintainability. +- Favor dependency injection and single-responsibility functions. +- Reduce duplication across CLI commands and deployment workflows. +- Keep CLI as source of truth; infra/docker is canonical container source. +- Keep Helm files generated at deploy time from docker sources. + +## Scope +- In scope: `src/cli/**`, `infra/**`. +- Out of scope: application runtime outside CLI, config loader, runtime services, API code. + +## Constraints +- No network access required. +- Use CLI for orchestration; infra scripts are helpers where needed. +- Keep backward-compatible commands where feasible. + +## Baseline Tests (added before changes) +- Unit tests under `tests/unit/cli` to lock in existing behavior for: + - Connection string parsing/merging in `src/cli/commands/db_utils.py`. + - Compose command construction helpers (to be introduced) and/or existing command utilities. + - Error handling wrappers in `src/cli/shared/console.py`. + +## Phases + +### Phase 1: Hygiene and Dependency Cleanup +**Intent:** Eliminate dead code and tighten imports without behavior change. +- Remove dead module: `src/cli/shared/helpers.py` (commented-out stub). +- Fix import cycle: `src/cli/deployment/status_display.py` should import `CLIConsole` from `src/cli/shared/console.py`. +- Normalize dev cleanup script: rewrite `infra/docker/dev/cleanup_dev.sh` into a single script. +- De-duplicate unused scripts: drop `infra/scripts/entrypoint.sh` if not referenced. + +**Verification:** run `tests/unit/cli`. + +### Phase 2: CLI Context & DI +**Intent:** Centralize runtime dependencies for better testability. +- Introduce `src/cli/context.py` with a `CLIContext` (console, project_root, shell commands, k8s controller). +- Add Typer callback in `src/cli/__init__.py` to attach context. +- Remove module-level globals in `src/cli/commands/k8s.py` and `src/cli/commands/k8s_db.py`; pull from context. + +**Verification:** run `tests/unit/cli`. + +### Phase 3: Compose Runner + Command Slimming +**Intent:** Replace ad-hoc subprocess calls with shared helpers. +- Add `src/cli/shared/compose.py` (or `src/cli/deployment/compose.py`) with helper methods for compose invocations. +- Refactor `src/cli/commands/dev.py` and `src/cli/commands/prod.py` to use the compose helper. +- Normalize service maps in one place. + +**Verification:** run `tests/unit/cli`. + +### Phase 4: DB Workflow Consolidation +**Intent:** DRY up prod/k8s DB commands. +- Add `src/cli/commands/db/` package: + - `runtime.py` (protocol/adapter interface) + - `runtime_compose.py` + - `runtime_k8s.py` + - `workflows.py` (shared flows: create/init/verify/sync/backup/reset/status/migrate) +- Convert `src/cli/commands/prod_db.py` and `src/cli/commands/k8s_db.py` into thin CLI wrappers. + +**Verification:** run `tests/unit/cli`. + +### Phase 5: Deployer Base Consolidation +**Intent:** Remove duplication across deployers. +- Move shared data subdir list to `src/cli/deployment/constants.py`. +- Factor container restart logic into `BaseDeployer`. +- Expose public helper for ensuring data directories (avoid private access). + +**Verification:** run `tests/unit/cli`. + +### Phase 6: Infra Consistency +**Intent:** Ensure single source of truth for deployment artifacts. +- Keep docker sources canonical; ensure `ConfigSynchronizer` only copies from `infra/docker/prod`. +- Remove unused duplicate `verify-init.sh` in `infra/docker/prod/postgres/admin-scripts` or replace with wrapper. +- Fix import order in `infra/docker/dev/keycloak/setup_script.py`. +- Align `infra/docker/dev/setup_dev.sh` to call CLI (or mark as legacy). + +**Verification:** run `tests/unit/cli`. + +### Phase 7: Entity CLI Split +**Intent:** Shorten and decouple long CLI modules. +- Split `src/cli/commands/entity.py` into `src/cli/commands/entity/` package. +- Add small helper to safely insert/remove router registrations. + +**Verification:** run `tests/unit/cli`. + +## Milestones & Rollback +- Each phase is independently shippable. +- If a phase breaks tests, revert only the phase’s changes and re-approach with smaller steps. + +## Test Command +- `uv run pytest tests/unit/cli` diff --git a/docs/database-migrations.md b/docs/database-migrations.md new file mode 100644 index 0000000..884c3b4 --- /dev/null +++ b/docs/database-migrations.md @@ -0,0 +1,564 @@ +# Database Migrations Guide + +This guide covers database schema migrations using Alembic, integrated into the API Forge CLI for both bundled and external PostgreSQL deployments. + +## Overview + +API Forge uses **Alembic** for database schema migrations with automatic model discovery from SQLModel table definitions. The system: + +- **Auto-discovers** all SQLModel tables - no manual import lists to maintain +- **Handles port-forwarding** automatically for bundled Kubernetes PostgreSQL +- **Supports both** bundled (in-cluster) and external PostgreSQL databases +- **Integrates** with the CLI for a seamless workflow + +## Architecture + +### Components + +- **`alembic.ini`** - Alembic configuration file (project root) +- **`migrations/env.py`** - Migration environment with dynamic model discovery +- **`migrations/versions/`** - Individual migration scripts +- **`src/app/entities/loader.py`** - Dynamic table discovery using `SQLModel.metadata` +- **CLI commands** - `uv run api-forge-cli k8s db migrate ...` + +### How It Works + +1. **Model Registration**: All SQLModel classes with `table=True` automatically register with `SQLModel.metadata` when imported +2. **Dynamic Discovery**: `src/app/entities/loader.py` uses `rglob("table.py")` to find all table modules and imports them +3. **Autogeneration**: Alembic compares `SQLModel.metadata` (your models) against the database schema to detect changes +4. **Port-Forwarding**: CLI automatically establishes `kubectl port-forward` for bundled PostgreSQL before running Alembic + +## Quick Start + +### 1. Create Your First Migration + +After defining your SQLModel tables: + +```bash +# Auto-generate migration from model changes +uv run api-forge-cli k8s db migrate revision "initial schema" --autogenerate +``` + +This will: +- Scan all `table.py` files in `src/app/entities/` +- Compare models against the database schema +- Generate a migration file in `migrations/versions/` + +### 2. Review the Generated Migration + +```bash +# Check what was generated +cat migrations/versions/2025*_initial_schema.py +``` + +Review the `upgrade()` and `downgrade()` functions to ensure correctness. + +### 3. Apply the Migration + +```bash +# Apply to database +uv run api-forge-cli k8s db migrate upgrade +``` + +### 4. Verify + +```bash +# Check current migration state +uv run api-forge-cli k8s db migrate current +``` + +## CLI Commands + +All commands work with both bundled and external PostgreSQL. + +### Create Migrations + +```bash +# Auto-generate from model changes (recommended) +uv run api-forge-cli k8s db migrate revision "add user email" --autogenerate + +# Create empty template for manual SQL +uv run api-forge-cli k8s db migrate revision "custom index" --no-autogenerate + +# Generate SQL without applying +uv run api-forge-cli k8s db migrate upgrade --sql > migration.sql +``` + +### Apply Migrations + +```bash +# Apply all pending migrations +uv run api-forge-cli k8s db migrate upgrade + +# Apply to specific revision +uv run api-forge-cli k8s db migrate upgrade abc123 + +# Apply one migration at a time +uv run api-forge-cli k8s db migrate upgrade +1 +``` + +### Rollback Migrations + +```bash +# Rollback to specific revision +uv run api-forge-cli k8s db migrate downgrade abc123 + +# Rollback one migration +uv run api-forge-cli k8s db migrate downgrade -1 + +# Rollback all (to empty database) +uv run api-forge-cli k8s db migrate downgrade base + +# Generate rollback SQL without applying +uv run api-forge-cli k8s db migrate downgrade -1 --sql > rollback.sql +``` + +### View Migration State + +```bash +# Show current migration version +uv run api-forge-cli k8s db migrate current + +# Show current head revision(s) +uv run api-forge-cli k8s db migrate heads + +# Show a specific migration's details +uv run api-forge-cli k8s db migrate show 19becf30b774 + +# Show all migrations +uv run api-forge-cli k8s db migrate history + +# Show detailed history with verbose flag +uv run api-forge-cli k8s db migrate history --verbose +``` + +### Team Workflows (Multiple Heads) + +When multiple developers generate migrations in parallel, Alembic can end up with +multiple heads. These commands help resolve that cleanly. + +```bash +# View current heads +uv run api-forge-cli k8s db migrate heads + +# Merge all current heads into a single head +uv run api-forge-cli k8s db migrate merge --message "merge heads" + +# Merge specific revisions +uv run api-forge-cli k8s db migrate merge --message "merge" -r abc123 -r def456 +``` + +### Stamping (Baseline / Repair) + +`stamp` sets the database's Alembic revision *without* running migrations. +This is useful for baselining an existing database or repairing the version table. + +```bash +# Mark DB as up-to-date with the latest migration +uv run api-forge-cli k8s db migrate stamp head + +# Mark DB as a specific revision +uv run api-forge-cli k8s db migrate stamp 19becf30b774 +``` + +## Development Workflow + +### Adding a New Entity + +When you add a new entity to your application: + +1. **Create the table model** in `src/app/entities///table.py`: + +```python +from sqlmodel import Field +from src.app.entities.core._base import EntityTable + +class ProductTable(EntityTable, table=True): + """Product table model.""" + + name: str = Field(max_length=255) + price: float = Field(gt=0) + sku: str = Field(max_length=100, index=True) +``` + +2. **Generate migration** (auto-detected, no imports needed): + +```bash +uv run api-forge-cli k8s db migrate revision "add product table" --autogenerate +``` + +3. **Review** the generated migration file + +4. **Apply** to database: + +```bash +uv run api-forge-cli k8s db migrate upgrade +``` + +That's it! The dynamic loader finds your new `table.py` automatically. + +### Modifying Existing Tables + +1. **Update the table model** in `src/app/entities///table.py`: + +```python +class UserTable(EntityTable, table=True): + # ... existing fields ... + + # Add new field + phone_number: str | None = Field(default=None, max_length=20) +``` + +2. **Generate migration**: + +```bash +uv run api-forge-cli k8s db migrate revision "add user phone number" --autogenerate +``` + +3. **Review and apply**: + +```bash +cat migrations/versions/*_add_user_phone_number.py +uv run api-forge-cli k8s db migrate upgrade +``` + +### Testing Migrations + +Before applying to production: + +1. **Apply migration** in development/staging: + +```bash +uv run api-forge-cli k8s db migrate upgrade +``` + +2. **Test the application** with the new schema + +3. **Test rollback**: + +```bash +uv run api-forge-cli k8s db migrate downgrade -1 +# Verify app still works +uv run api-forge-cli k8s db migrate upgrade +``` + +## Best Practices + +### 1. Always Use Autogenerate + +Let Alembic detect changes automatically: + +```bash +# ✅ Recommended +uv run api-forge-cli k8s db migrate revision "add column" --autogenerate + +# ❌ Avoid (unless you need custom SQL) +uv run api-forge-cli k8s db migrate revision "add column" --no-autogenerate +``` + +### 2. Review Generated Migrations + +Alembic may not always generate perfect migrations. Always review: + +```python +def upgrade() -> None: + # Check for: + # - Correct column types + # - Proper null/default handling + # - Index creation + # - Foreign key constraints + op.add_column('users', sa.Column('email', sa.String(255), nullable=True)) + # Add data migration if needed + op.execute("UPDATE users SET email = concat(username, '@example.com')") + # Then enforce constraint + op.alter_column('users', 'email', nullable=False) +``` + +### 3. Write Reversible Migrations + +Always implement proper `downgrade()`: + +```python +def upgrade() -> None: + op.add_column('users', sa.Column('status', sa.String(20))) + +def downgrade() -> None: + op.drop_column('users', 'status') +``` + +### 4. Test Migrations Before Deploying + +```bash +# Apply migration +uv run api-forge-cli k8s db migrate upgrade + +# Test app functionality + +# Test rollback +uv run api-forge-cli k8s db migrate downgrade -1 + +# Test app still works + +# Re-apply +uv run api-forge-cli k8s db migrate upgrade +``` + +### 5. Handle Data Migrations Carefully + +For operations that modify existing data: + +```python +def upgrade() -> None: + # 1. Add column as nullable + op.add_column('products', sa.Column('category', sa.String(50), nullable=True)) + + # 2. Populate data + op.execute(""" + UPDATE products + SET category = 'general' + WHERE category IS NULL + """) + + # 3. Make non-nullable + op.alter_column('products', 'category', nullable=False) +``` + +### 6. Never Edit Applied Migrations + +Once a migration is applied to any environment (dev, staging, prod): + +- ❌ Never edit it +- ✅ Create a new migration to fix issues + +### 7. Keep Migrations Small + +- One logical change per migration +- Makes rollback easier +- Simplifies code review + +```bash +# ✅ Good - focused migrations +uv run api-forge-cli k8s db migrate revision "add user email" +uv run api-forge-cli k8s db migrate revision "add email index" + +# ❌ Bad - too many changes +uv run api-forge-cli k8s db migrate revision "update user schema" +``` + +## Production Deployment + +### Pre-Deployment + +1. **Generate migration** in development: + +```bash +uv run api-forge-cli k8s db migrate revision "production change" --autogenerate +``` + +2. **Test thoroughly** in staging environment + +3. **Commit migration file** to version control + +### Deployment Process + +1. **Backup database**: + +```bash +kubectl exec -n api-forge-prod postgresql-0 -- pg_dump -U postgres appdb > backup.sql +``` + +2. **Apply migration**: + +```bash +uv run api-forge-cli k8s db migrate upgrade +``` + +3. **Verify**: + +```bash +uv run api-forge-cli k8s db migrate current +uv run api-forge-cli k8s db verify +``` + +4. **Deploy application** with new code + +### Rollback Plan + +If issues occur: + +```bash +# 1. Rollback application deployment +kubectl rollout undo deployment/api-forge -n api-forge-prod + +# 2. Rollback migration +uv run api-forge-cli k8s db migrate downgrade -1 + +# 3. Verify +uv run api-forge-cli k8s db verify +``` + +## Troubleshooting + +### Migration Fails with "target database has pending upgrade operations" + +**Cause**: Previous migration was interrupted + +**Solution**: +```bash +# Check current state +uv run api-forge-cli k8s db migrate current + +# Force to specific revision +uv run api-forge-cli k8s db migrate upgrade abc123 +``` + +### Alembic Can't Detect My New Table + +**Cause**: Table model not being imported + +**Solution**: Verify your `table.py` file exists in `src/app/entities/`: + +```bash +# Should list your table.py files +find src/app/entities -name "table.py" + +# Test import +uv run python -c "from src.app.entities.loader import get_metadata; print(get_metadata().tables.keys())" +``` + +### Port-Forward Connection Error + +**Cause**: Bundled PostgreSQL pod not ready + +**Solution**: +```bash +# Check pod status +kubectl get pods -n api-forge-prod -l app=postgresql + +# Restart port-forward by retrying command +uv run api-forge-cli k8s db migrate current +``` + +### Schema Drift Detected + +**Cause**: Manual changes made to database outside migrations + +**Solution**: +```bash +# Generate migration to align +uv run api-forge-cli k8s db migrate revision "fix schema drift" --autogenerate + +# Review carefully - may need manual editing +cat migrations/versions/*_fix_schema_drift.py +``` + +### Multiple Heads Detected + +**Cause**: Branches in migration history (multiple developers) + +**Solution**: +```bash +# View heads +uv run api-forge-cli k8s db migrate heads + +# Merge branches +uv run api-forge-cli k8s db migrate revision "merge branches" --merge +``` + +## Advanced Topics + +### Branching and Merging + +For teams working on multiple features: + +```bash +# Create branch label +uv run api-forge-cli k8s db migrate revision "feature a" --autogenerate --branch-label feature_a + +# Create another branch +uv run api-forge-cli k8s db migrate revision "feature b" --autogenerate --branch-label feature_b + +# Merge branches +uv run api-forge-cli k8s db migrate revision "merge features" --merge +``` + +### Custom SQL Migrations + +For complex operations not detectable by autogenerate: + +```bash +# Create empty template +uv run api-forge-cli k8s db migrate revision "optimize indexes" --no-autogenerate +``` + +Edit the generated file: + +```python +def upgrade() -> None: + # Custom SQL + op.execute(""" + CREATE INDEX CONCURRENTLY idx_users_email_lower + ON users (LOWER(email)) + """) + +def downgrade() -> None: + op.execute("DROP INDEX idx_users_email_lower") +``` + +### Offline SQL Generation + +Generate SQL without database connection: + +```bash +# Generate upgrade SQL +uv run api-forge-cli k8s db migrate upgrade --sql > upgrade.sql + +# Apply manually +psql -h localhost -U postgres appdb < upgrade.sql +``` + +## Reference + +### Environment Variables + +Set by CLI automatically, but available for manual use: + +- `DATABASE_URL` - Complete PostgreSQL connection string (set by CLI) + +### File Structure + +``` +project/ +├── alembic.ini # Alembic config +├── migrations/ +│ ├── env.py # Migration environment +│ ├── script.py.mako # Migration template +│ ├── README.md # Technical reference +│ └── versions/ # Migration scripts +│ └── 20251224_0327_initial.py +└── src/ + └── app/ + └── entities/ + ├── loader.py # Dynamic table discovery + └── core/ + └── user/ + └── table.py # User table model +``` + +### Related Documentation + +- [PostgreSQL Configuration](postgres/configuration.md) +- [Kubernetes Database Management](fastapi-kubernetes-deployment.md) +- [Production Deployment](infra/PRODUCTION_DEPLOYMENT.md) +- [Database Security](postgres/security.md) + +## Getting Help + +If you encounter issues: + +1. Check this guide's **Troubleshooting** section +2. Review `migrations/README.md` for technical details +3. Check Alembic logs for detailed error messages +4. Verify database connectivity: `uv run api-forge-cli k8s db verify` + +For Alembic-specific documentation: https://alembic.sqlalchemy.org/ diff --git a/docs/fastapi-kubernetes-deployment.md b/docs/fastapi-kubernetes-deployment.md index 108770d..a32e5ef 100644 --- a/docs/fastapi-kubernetes-deployment.md +++ b/docs/fastapi-kubernetes-deployment.md @@ -110,6 +110,76 @@ infra/helm/api-forge/ - **Automatic Sync**: CLI synchronizes settings before each deployment - **Timestamp Annotations**: Forces pod recreation to ensure latest Docker images +## Database Management + +The CLI provides comprehensive database management commands for Kubernetes deployments, supporting both bundled PostgreSQL (deployed in the cluster) and external databases (like Aiven, AWS RDS, Google Cloud SQL). + +### Database Setup Commands + +```bash +# Initialize database with roles, schemas, and permissions +uv run api-forge-cli k8s db init + +# Verify database configuration and test authentication +uv run api-forge-cli k8s db verify + +# Synchronize local password files to database (after password changes) +uv run api-forge-cli k8s db sync + +# Check database health and performance metrics +uv run api-forge-cli k8s db status + +# Create a backup of the database +uv run api-forge-cli k8s db backup + +# Reset database to clean state (DESTRUCTIVE - dev/test only) +uv run api-forge-cli k8s db reset +``` + +### Using External PostgreSQL (Aiven, RDS, Cloud SQL) + +To use an external managed PostgreSQL database instead of the bundled one: + +1. **Configure the external database**: + ```bash + # Using connection string + uv run api-forge-cli k8s db create --external \ + --connection-string "postgres://admin:secret@db.example.com:5432/mydb?sslmode=require" + + # Or using individual parameters + uv run api-forge-cli k8s db create --external \ + --host db.aivencloud.com --port 20369 \ + --username avnadmin --password secret \ + --database defaultdb --sslmode require + ``` + + This command will: + - Update `.env` with `PRODUCTION_DATABASE_URL` + - Configure database credentials in `config.yaml` + - Generate necessary password files in `infra/secrets/keys/` + +2. **Initialize the database** (creates roles, schemas, grants permissions): + ```bash + uv run api-forge-cli k8s db init + ``` + +3. **Verify the setup** (tests connectivity and credentials): + ```bash + uv run api-forge-cli k8s db verify + ``` + +4. **Deploy** - the application will automatically use the external database: + ```bash + uv run api-forge-cli deploy up k8s + ``` + +**Important Notes:** +- The `init` command creates application users (`appuser`, `backupuser`, `temporaluser`) and the `app` schema +- The `verify` command now tests password authentication to catch mismatches early +- The `sync` command updates database passwords to match your local secret files +- In production, the app automatically uses `search_path=app` to isolate tables from the `public` schema +- Connection strings preserve existing query parameters (like `?sslmode=require`) while adding production settings + ## Deployment Steps ### Step 1: Build Docker Images diff --git a/docs/fastapi-production-deployment-docker-compose.md b/docs/fastapi-production-deployment-docker-compose.md index b28bf4d..8221a7c 100644 --- a/docs/fastapi-production-deployment-docker-compose.md +++ b/docs/fastapi-production-deployment-docker-compose.md @@ -27,6 +27,13 @@ Deploy to production with Docker Compose: cd infra/secrets ./generate_secrets.sh +# For external databases (optional): Initialize database with roles/schema +# Note: Database management commands are primarily for Kubernetes deployments +# For Docker Compose, the bundled PostgreSQL container handles initialization automatically +# If using an external database, you can use the k8s db commands: +uv run api-forge-cli k8s db init # One-time setup for external DB +uv run api-forge-cli k8s db verify # Verify external DB configuration + # Copy deterministic secret template and fill OIDC client secrets, webhook tokens, etc. cp user-provided.env.example user-provided.env # Edit user-provided.env with production-only values (not committed to git) diff --git a/docs/index.md b/docs/index.md index 7fc450d..e5aea10 100644 --- a/docs/index.md +++ b/docs/index.md @@ -160,6 +160,18 @@ Production-ready Kubernetes manifests with: [Deploy to Kubernetes →](./fastapi-kubernetes-deployment.md) +### Database Migrations + +Integrated Alembic migrations with automatic model discovery: + +- Auto-detects all SQLModel tables +- Port-forwarding support for Kubernetes PostgreSQL +- CLI integration for bundled and external databases +- Autogeneration from model changes +- Production-safe rollback support + +[Manage database migrations →](./database-migrations.md) + ### Temporal Workflows Built-in support for distributed workflows: @@ -182,6 +194,12 @@ uv run api-forge-cli deploy down dev # Stop services uv run api-forge-cli deploy status dev # Check service status uvicorn src_main:app --reload # Start FastAPI server +# Database migrations +uv run api-forge-cli k8s db migrate upgrade # Apply migrations +uv run api-forge-cli k8s db migrate revision "message" # Create migration +uv run api-forge-cli k8s db migrate current # Show current state +uv run api-forge-cli k8s db migrate history # View history + # Entity generation uv run api-forge-cli entity add User # Generate entity scaffold uv run api-forge-cli entity list # List entities @@ -251,4 +269,5 @@ MIT License - see [LICENSE](../LICENSE) for details. 1. **[Set up your development environment](./fastapi-docker-dev-environment.md)** 2. **[Understand authentication](./fastapi-auth-oidc-bff.md)** 3. **[Explore the architecture](./fastapi-clean-architecture-overview.md)** -4. **[Deploy to production](./fastapi-kubernetes-deployment.md)** +4. **[Manage database migrations](./database-migrations.md)** +5. **[Deploy to production](./fastapi-kubernetes-deployment.md)** diff --git a/docs/infra/PRODUCTION_DEPLOYMENT.md b/docs/infra/PRODUCTION_DEPLOYMENT.md index a7a2e05..f899526 100644 --- a/docs/infra/PRODUCTION_DEPLOYMENT.md +++ b/docs/infra/PRODUCTION_DEPLOYMENT.md @@ -9,7 +9,7 @@ This guide covers the deployment of the FastAPI application stack to production ### Production Stack - **Application**: FastAPI with OIDC authentication, session management, and rate limiting -- **Database**: PostgreSQL 16 with SSL, backup automation, and performance tuning +- **Database**: PostgreSQL 16 with SSL, backup automation, and performance tuning (bundled or external) - **Cache/Sessions**: Redis 7 with persistence, security hardening, and memory optimization - **Workflows**: Temporal Server with PostgreSQL backend for reliable workflow execution - **Reverse Proxy**: Nginx with SSL termination, security headers, and load balancing @@ -65,6 +65,11 @@ uv run api-forge-cli deploy status prod **Kubernetes (cluster deployment):** ```bash +# For external databases: Initialize first +uv run api-forge-cli k8s db init # Creates roles, schemas, permissions +uv run api-forge-cli k8s db verify # Verifies setup and credentials + +# Deploy the application uv run api-forge-cli deploy up k8s # Check deployment status @@ -72,6 +77,11 @@ uv run api-forge-cli deploy status k8s # View release history uv run api-forge-cli deploy history + +# Database management (Kubernetes only) +uv run api-forge-cli k8s db sync # Sync password files to database +uv run api-forge-cli k8s db status # Check database health +uv run api-forge-cli k8s db backup # Create backup ``` ### 4. Verify Deployment @@ -89,6 +99,35 @@ kubectl logs -n api-forge-prod -l app.kubernetes.io/name=app -f ## 🔧 Configuration +### Database Options + +API Forge supports two PostgreSQL deployment modes: + +1. **Bundled PostgreSQL** (default for Docker Compose/Kubernetes): + - Automatically deployed as part of the stack + - Handled by Docker Compose or Helm chart + - No external setup required + +2. **External PostgreSQL** (Aiven, AWS RDS, Google Cloud SQL, Azure Database, etc.): + - Use managed database service for better scalability and reliability + - Configure with `uv run api-forge-cli k8s db create --external` + - Initialize with `uv run api-forge-cli k8s db init` + - Verify with `uv run api-forge-cli k8s db verify` + +**Example external database setup:** +```bash +# Configure external database with connection string +uv run api-forge-cli k8s db create --external \ + --connection-string "postgres://avnadmin@your-db.aivencloud.com:20369/defaultdb?sslmode=require" + +# Or with individual parameters +uv run api-forge-cli k8s db create --external \ + --host your-db.aivencloud.com --port 20369 \ + --username avnadmin --password your-password \ + --database defaultdb --sslmode require +``` + + ### Environment Variables #### Application Configuration diff --git a/infra/docker/dev/cleanup_dev.sh b/infra/docker/dev/cleanup_dev.sh index f26efe6..58eb222 100755 --- a/infra/docker/dev/cleanup_dev.sh +++ b/infra/docker/dev/cleanup_dev.sh @@ -1,7 +1,7 @@ #!/bin/bash # Development environment cleanup script -# This script stops and removes the Keycloak container and data +# Stops containers and optionally removes data volumes. set -e @@ -13,29 +13,12 @@ cd "$DEV_DIR" # Stop and remove containers echo "🛑 Stopping containers..." -docker-compose down - -#!/bin/bash - -# Development environment cleanup script -# This script stops and removes containers and optionally removes data volumes - -set -e - -DEV_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -echo "🧹 Cleaning up development environment..." - -cd "$DEV_DIR" - -# Stop and remove containers -echo "🛑 Stopping containers..." -docker-compose down +docker compose down # Remove volumes (optional) if [ "$1" = "--remove-data" ]; then echo "🗑️ Removing persistent data volumes..." - docker-compose down -v + docker compose down -v docker volume rm dev-env_keycloak_data dev-env_postgres_data 2>/dev/null || true fi @@ -43,8 +26,3 @@ echo "✅ Development environment cleaned up!" echo "" echo "To restart the environment:" echo " ./setup_dev.sh" - -echo "✅ Development environment cleaned up!" -echo "" -echo "To restart the environment:" -echo " ./setup_dev.sh" \ No newline at end of file diff --git a/infra/docker/dev/keycloak/setup_script.py b/infra/docker/dev/keycloak/setup_script.py index 9d35bc7..b1dcce1 100644 --- a/infra/docker/dev/keycloak/setup_script.py +++ b/infra/docker/dev/keycloak/setup_script.py @@ -5,11 +5,11 @@ import sys import time -from src.dev.setup_keycloak import KeycloakSetup - # Add the src directory to the Python path sys.path.insert(0, "/app/src") +from src.dev.setup_keycloak import KeycloakSetup + try: # Use the container's internal hostname for Keycloak keycloak_url = os.getenv("KEYCLOAK_URL", "http://keycloak:8080") diff --git a/infra/docker/dev/setup_dev.sh b/infra/docker/dev/setup_dev.sh index dbcd4eb..b08a402 100755 --- a/infra/docker/dev/setup_dev.sh +++ b/infra/docker/dev/setup_dev.sh @@ -1,13 +1,21 @@ #!/bin/bash # Development environment setup script -# This script sets up a local Keycloak instance with test realm and client configuration +# Prefer the CLI as the source of truth, fall back to direct docker compose. set -e DEV_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$DEV_DIR")" +if command -v uv >/dev/null 2>&1; then + echo "🚀 Using API Forge CLI to start dev environment..." + (cd "$PROJECT_ROOT" && uv run api-forge-cli dev up --no-start-server) + exit 0 +fi + +echo "⚠️ uv not found; falling back to docker compose setup" + echo "🚀 Setting up development environment..." # Check if Docker is running @@ -19,7 +27,7 @@ fi # Start development services echo "📦 Starting development services..." cd "$DEV_DIR" -docker-compose up -d +docker compose up -d # Wait for Keycloak to be ready echo "⏳ Waiting for Keycloak to be ready..." @@ -60,5 +68,5 @@ echo " - testuser1@example.com / password123" echo " - testuser2@example.com / password123" echo "" echo "To stop the environment:" -echo " cd dev_env && docker-compose down" -echo "" \ No newline at end of file +echo " cd dev_env && docker compose down" +echo "" diff --git a/infra/docker/prod/postgres/admin-scripts/verify-init.sh b/infra/docker/prod/postgres/admin-scripts/verify-init.sh deleted file mode 100644 index 1acd29c..0000000 --- a/infra/docker/prod/postgres/admin-scripts/verify-init.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/sh -set -eu - -APP_DB="${APP_DB:-${APP_DB:-appdb}}" -APP_USER="${APP_USER:-${APP_USER:-appuser}}" -APP_MIGRATION="${APP_MIGRATION:-${APP_MIGRATION:-backupuser}}" -APP_SCHEMA="${APP_SCHEMA:-app}" - -# superuser psql (socket); add -h 127.0.0.1 if needed -PSQL_SUPER="psql -v ON_ERROR_STOP=1 -U postgres" -PSQL_APP="$PSQL_SUPER -d $APP_DB" - -ok(){ printf "✅ %s\n" "$*"; } -bad(){ printf "❌ %s\n" "$*"; failed=1; } - -failed=0 - -# helper: run psql -c with one inline \set -pset() { - var="$1"; val="$2"; shift 2 - # delim newlines so \set is in the same session as the query - $PSQL_SUPER -At -c "$(printf "\\set %s '%s'\n%s" "$var" "$val" "$*")" -} - -# ---- roles LOGIN ---- -pset r "$APP_USER" "SELECT rolcanlogin FROM pg_authid WHERE rolname = :'r';" | grep -qx t \ - && ok "Role $APP_USER has LOGIN" || bad "Role $APP_USER missing LOGIN" - -pset r "$APP_MIGRATION" "SELECT rolcanlogin FROM pg_authid WHERE rolname = :'r';" | grep -qx t \ - && ok "Role $APP_MIGRATION has LOGIN" || bad "Role $APP_MIGRATION missing LOGIN" - -# ---- database owner ---- -db_owner="$(pset db "$APP_DB" "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = :'db';" || true)" -[ -z "$db_owner" ] && bad "Database $APP_DB does not exist" || { - [ "$db_owner" = "$APP_USER" ] && ok "Database $APP_DB owner is $APP_USER" \ - || bad "Database $APP_DB owner is '$db_owner' (expected $APP_USER)" -} - -# ---- schema owner (in app DB) ---- -schema_owner="$($PSQL_APP -At -c "$(printf "\\set s '%s'\n%s" "$APP_SCHEMA" \ -"SELECT r.rolname - FROM pg_namespace n JOIN pg_roles r ON r.oid = n.nspowner - WHERE n.nspname = :'s';")" || true)" - -[ -z "$schema_owner" ] && bad "Schema $APP_SCHEMA does not exist in $APP_DB" || { - [ "$schema_owner" = "$APP_USER" ] && ok "Schema $APP_SCHEMA owner is $APP_USER" \ - || bad "Schema $APP_SCHEMA owner is '$schema_owner' (expected $APP_USER)" -} - -# ---- schema privileges ---- -for who in "$APP_MIGRATION:USAGE" "$APP_MIGRATION:CREATE" "$APP_USER:USAGE"; do - role="${who%:*}"; priv="${who#*:}" - $PSQL_APP -At -c "$(printf "\\set r '%s'\n\\set s '%s'\n\\set p '%s'\nSELECT has_schema_privilege(:'r', :'s', :'p');" "$role" "$APP_SCHEMA" "$priv")" \ - | grep -qx t \ - && ok "$role has $priv on $APP_SCHEMA" \ - || bad "$role missing $priv on $APP_SCHEMA" -done - -# ---- DML on existing tables ---- -missing_tbls="$($PSQL_APP -At -c "$(printf "\\set s '%s'\n\\set u '%s'\n%s" "$APP_SCHEMA" "$APP_USER" \ -"WITH t AS ( - SELECT quote_ident(n.nspname)||'.'||quote_ident(c.relname) AS fq - FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = :'s' AND c.relkind IN ('r','p') - ), - missing AS ( - SELECT fq FROM t - WHERE NOT has_table_privilege(:'u', fq, 'SELECT') - OR NOT has_table_privilege(:'u', fq, 'INSERT') - OR NOT has_table_privilege(:'u', fq, 'UPDATE') - OR NOT has_table_privilege(:'u', fq, 'DELETE') - ) - SELECT COALESCE(string_agg(fq, ', '), '') FROM missing;")" || true)" - -[ -z "$missing_tbls" ] \ - && ok "$APP_USER has DML on all tables in $APP_SCHEMA" \ - || bad "$APP_USER missing DML on: $missing_tbls" - -# ---- sequences ---- -missing_seqs="$($PSQL_APP -At -c "$(printf "\\set s '%s'\n\\set u '%s'\n%s" "$APP_SCHEMA" "$APP_USER" \ -"WITH seqs AS ( - SELECT quote_ident(n.nspname)||'.'||quote_ident(c.relname) AS fq - FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = :'s' AND c.relkind = 'S' - ), - missing AS ( - SELECT fq FROM seqs - WHERE NOT has_sequence_privilege(:'u', fq, 'USAGE') - OR NOT has_sequence_privilege(:'u', fq, 'SELECT') - ) - SELECT COALESCE(string_agg(fq, ', '), '') FROM missing;")" || true)" - -[ -z "$missing_seqs" ] \ - && ok "$APP_USER has USAGE,SELECT on all sequences in $APP_SCHEMA" \ - || bad "$APP_USER missing sequence privs on: $missing_seqs" - -# ---- summary ---- -[ "${failed:-0}" -eq 0 ] && { echo "✅ Verification passed"; exit 0; } \ - || { echo "❌ Verification failed"; exit 1; } diff --git a/infra/docker/prod/temporal/scripts/entrypoint.sh b/infra/docker/prod/temporal/scripts/entrypoint.sh index 8ae192b..69628a8 100755 --- a/infra/docker/prod/temporal/scripts/entrypoint.sh +++ b/infra/docker/prod/temporal/scripts/entrypoint.sh @@ -14,5 +14,51 @@ else exit 1 fi +# Parse POSTGRES_SEEDS, SQL_HOST_NAME, and DB_PORT from PRODUCTION_DATABASE_URL if not set +# Supports: +# - postgresql://host:port/db?options (no credentials) +# - postgresql://host:port?options (no credentials, no db) +# - postgresql://user:pass@host:port/db?options (with credentials) +if [ -z "${POSTGRES_SEEDS:-}" ] && [ -n "${PRODUCTION_DATABASE_URL:-}" ]; then + echo "Parsing database connection from PRODUCTION_DATABASE_URL..." + + # Check if URL contains @ (has credentials) + if echo "$PRODUCTION_DATABASE_URL" | grep -q '@'; then + # Format: postgresql://user:pass@host:port/db + HOST_PORT=$(echo "$PRODUCTION_DATABASE_URL" | sed -E 's|.*@([^/?]+).*|\1|') + else + # Format: postgresql://host:port/db or postgresql://host:port?options + # Remove scheme (postgresql://) and everything after / or ? + HOST_PORT=$(echo "$PRODUCTION_DATABASE_URL" | sed -E 's|^[^:]+://([^/?]+).*|\1|') + fi + + # Extract just host (before :) + PG_HOST=$(echo "$HOST_PORT" | sed -E 's|:.*||') + # Extract port if present (after :), default to 5432 + PARSED_PORT=$(echo "$HOST_PORT" | grep -oE ':[0-9]+' | tr -d ':' || echo "") + + export POSTGRES_SEEDS="$PG_HOST" + echo "✓ Set POSTGRES_SEEDS=$POSTGRES_SEEDS from URL" + + # Set SQL_HOST_NAME if not set (for TLS verification) + if [ -z "${SQL_HOST_NAME:-}" ]; then + export SQL_HOST_NAME="$PG_HOST" + echo "✓ Set SQL_HOST_NAME=$SQL_HOST_NAME from URL" + fi + + # Set DB_PORT if parsed from URL + if [ -n "$PARSED_PORT" ]; then + export DB_PORT="$PARSED_PORT" + echo "✓ Set DB_PORT=$DB_PORT from URL" + fi +fi + +# Validate POSTGRES_SEEDS is set +if [ -z "${POSTGRES_SEEDS:-}" ]; then + echo "❌ ERROR: POSTGRES_SEEDS not set" >&2 + echo " Set PG_HOST in .env or provide PRODUCTION_DATABASE_URL" >&2 + exit 1 +fi + # Call Temporal's original entrypoint exec /etc/temporal/entrypoint.sh "$@" \ No newline at end of file diff --git a/infra/docker/prod/temporal/scripts/schema-setup.sh b/infra/docker/prod/temporal/scripts/schema-setup.sh index 8fbc716..2723495 100755 --- a/infra/docker/prod/temporal/scripts/schema-setup.sh +++ b/infra/docker/prod/temporal/scripts/schema-setup.sh @@ -2,12 +2,45 @@ set -euo pipefail # ---- Required env ---- -: "${TEMPORAL_DB:?missing DB (database name, e.g. temporal)}" -: "${TEMPORAL_VIS_DB:?missing DB (database name, e.g. temporal_visibility)}" -: "${TEMPORAL_DB_USER:?missing PG_USER (e.g. temporal_user)}" +: "${TEMPORAL_DB:?missing TEMPORAL_DB (database name, e.g. temporal)}" +: "${TEMPORAL_VIS_DB:?missing TEMPORAL_VIS_DB (database name, e.g. temporal_visibility)}" +: "${TEMPORAL_DB_USER:?missing TEMPORAL_DB_USER (e.g. temporal_user)}" : "${PW_FILE:?missing PW_FILE (password file for temporal user)}" -: "${EP:?missing EP (Postgres host, e.g. postgres)}" +# ---- Parse EP (postgres host) from PRODUCTION_DATABASE_URL if not set ---- +# Supports: +# - postgresql://host:port/db?options (no credentials) +# - postgresql://host:port?options (no credentials, no db) +# - postgresql://user:pass@host:port/db?options (with credentials) +if [ -z "${EP:-}" ] && [ -n "${PRODUCTION_DATABASE_URL:-}" ]; then + echo "Parsing database host from PRODUCTION_DATABASE_URL..." + + # Check if URL contains @ (has credentials) + if echo "$PRODUCTION_DATABASE_URL" | grep -q '@'; then + # Format: postgresql://user:pass@host:port/db + HOST_PORT=$(echo "$PRODUCTION_DATABASE_URL" | sed -E 's|.*@([^/?]+).*|\1|') + else + # Format: postgresql://host:port/db or postgresql://host:port?options + # Remove scheme (postgresql://) and everything after / or ? + HOST_PORT=$(echo "$PRODUCTION_DATABASE_URL" | sed -E 's|^[^:]+://([^/?]+).*|\1|') + fi + + # Extract just host (before :) + EP=$(echo "$HOST_PORT" | sed -E 's|:.*||') + # Extract port if present (after :), default to 5432 + PARSED_PORT=$(echo "$HOST_PORT" | grep -oE ':[0-9]+' | tr -d ':' || echo "") + if [ -n "$PARSED_PORT" ]; then + PG_PORT="$PARSED_PORT" + fi + # Also set TLS_SERVER_NAME to match the actual host + if [ -z "${TLS_SERVER_NAME:-}" ]; then + TLS_SERVER_NAME="$EP" + fi + echo "Parsed from URL: EP=$EP, PG_PORT=${PG_PORT:-5432}, TLS_SERVER_NAME=$TLS_SERVER_NAME" +fi + +# Validate EP is set +: "${EP:?missing EP (set EP directly or provide PRODUCTION_DATABASE_URL)}" # Optional env PG_PORT="${PG_PORT:-5432}" @@ -25,12 +58,12 @@ echo "Password loaded successfully." # TLS (defaults; override via env if needed) -SSL_MODE="${SSL_MODE:-verify-ca}" # or 'require' / 'verify-full' +SSL_MODE="${SSL_MODE:-verify-ca}" TLS_ENABLE="${TLS_ENABLE:-true}" TLS_CA_FILE="${TLS_CA_FILE:-/run/secrets/postgres_server_ca}" TLS_SERVER_NAME="${TLS_SERVER_NAME:-postgres}" # MUST match a SAN in the server cert -export PGSSLMODE="${SSL_MODE:-verify-ca}" +export PGSSLMODE="${SSL_MODE}" # Verify TLS CA file exists if TLS is enabled if [ "$TLS_ENABLE" = "true" ]; then diff --git a/infra/helm/api-forge-bundled-postgres/Chart.yaml b/infra/helm/api-forge-bundled-postgres/Chart.yaml new file mode 100644 index 0000000..70336d8 --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: api-forge-bundled-postgres +description: Production-ready PostgreSQL database for API Forge applications +type: application +version: 1.0.0 +appVersion: "15" +keywords: + - postgresql + - database + - api-forge +home: https://github.com/piewared/api_project_template +sources: + - https://github.com/piewared/api_project_template +maintainers: + - name: API Forge Team diff --git a/infra/helm/api-forge-bundled-postgres/FEATURES.md b/infra/helm/api-forge-bundled-postgres/FEATURES.md new file mode 100644 index 0000000..2ad75da --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/FEATURES.md @@ -0,0 +1,277 @@ +# API Forge Bundled PostgreSQL - Feature Parity Documentation + +## Overview + +This standalone PostgreSQL chart provides complete feature parity with the original PostgreSQL deployment that was part of the main `api-forge` chart. It can be deployed independently for users who want to manage the database separately from the application. + +## Deployment Options + +### Option 1: Standalone Bundled PostgreSQL (This Chart) +```bash +uv run api-forge-cli k8s db create +``` +Deploys PostgreSQL using this chart with all production features. + +### Option 2: External Managed PostgreSQL +Skip this chart entirely and configure your application to connect to: +- AWS RDS +- Google Cloud SQL +- Azure Database for PostgreSQL +- Any other managed PostgreSQL service + +### Option 3: Bitnami PostgreSQL (Fallback) +The CLI will fall back to Bitnami if this chart is not available (deprecated path). + +## Complete Feature List + +### ✅ Core Database Features +- [x] **Custom PostgreSQL Image**: Uses `app_data_postgres_image` with production optimizations +- [x] **StatefulSet Deployment**: Ensures stable network identity and ordered scaling +- [x] **Configurable Replicas**: Default 1 replica (can be increased for HA) +- [x] **Application Database**: `appdb` with 3-role security pattern +- [x] **Temporal Databases**: Optional `temporal` and `temporal_visibility` databases +- [x] **Resource Management**: CPU/memory requests and limits + +### ✅ Storage & Persistence +- [x] **Data Persistence**: Dedicated PVC for PostgreSQL data (default 20Gi) +- [x] **Backup Persistence**: Separate PVC for backups (default 40Gi) +- [x] **Storage Class Configuration**: Supports custom storage classes +- [x] **Volume Claim Templates**: Automatic data volume provisioning per replica + +### ✅ Security Features +- [x] **TLS/SSL Encryption**: All connections encrypted with TLS 1.2+ +- [x] **Certificate Management**: Mounts server certificates from secrets +- [x] **CA Certificates**: Optional client certificate verification +- [x] **SCRAM-SHA-256 Authentication**: Modern password hashing +- [x] **Password Secrets**: All passwords stored in Kubernetes secrets +- [x] **3-Role Security Pattern**: + - `appowner` (NOLOGIN) - owns database/schema + - `appuser` (LOGIN) - runtime read/write operations + - `backupuser` (LOGIN) - read-only access +- [x] **Least Privilege**: Runtime users cannot drop database/schema +- [x] **Network Policy**: Optional pod-level access control (disabled by default, enable in production) +- [x] **Pod Security**: fsGroup configuration and seccomp profile + +### ✅ Configuration Management +- [x] **PostgreSQL Configuration**: Production-optimized `postgresql.conf` +- [x] **Access Control**: Hardened `pg_hba.conf` with TLS enforcement +- [x] **Initialization Scripts**: Automated database/role setup +- [x] **Environment Variables**: Comprehensive configuration via env vars +- [x] **ConfigMap Integration**: All config files in ConfigMap + +### ✅ High Availability & Reliability +- [x] **Pod Disruption Budget**: Ensures minimum 1 replica during maintenance +- [x] **Health Probes**: Liveness and readiness checks +- [x] **Ordered Updates**: RollingUpdate strategy with OrderedReady policy +- [x] **Resource Limits**: Prevents resource exhaustion +- [x] **Shared Memory**: Dedicated tmpfs for PostgreSQL shared buffers + +### ✅ Initialization & Setup +- [x] **Automatic Database Creation**: Creates `appdb` on first start +- [x] **Role Management**: Creates all required roles with correct permissions +- [x] **Schema Setup**: Creates application schema with proper ownership +- [x] **Permission Grants**: Sets up all runtime and default privileges +- [x] **Extension Installation**: Installs required extensions (`btree_gin`) +- [x] **Temporal Setup** (optional): Creates Temporal databases if enabled +- [x] **Idempotent Initialization**: Safe to run multiple times + +### ✅ Operational Features +- [x] **Structured Logging**: JSON logging with configurable retention +- [x] **Statement Tracking**: `pg_stat_statements` for query performance +- [x] **Connection Pooling Support**: Configurable max connections +- [x] **Backup Support**: Dedicated volume for pg_dump/pg_basebackup +- [x] **Monitoring Ready**: Exposes metrics for Prometheus integration +- [x] **Custom Entrypoint**: Universal entrypoint script with secret management + +## Configuration Values + +### Required Secrets + +The chart requires three secrets to be created before deployment: + +1. **postgres-secrets** - Database passwords + ```bash + kubectl create secret generic postgres-secrets \ + --from-file=postgres_password= \ + --from-file=postgres_app_owner_pw= \ + --from-file=postgres_app_user_pw= \ + --from-file=postgres_app_ro_pw= \ + --from-file=postgres_temporal_pw= \ + -n api-forge-prod + ``` + +2. **postgres-tls** - TLS certificates + ```bash + kubectl create secret generic postgres-tls \ + --from-file=server.crt= \ + --from-file=server.key= \ + -n api-forge-prod + ``` + +3. **postgres-ca** - CA certificate (optional, for client verification) + ```bash + kubectl create secret generic postgres-ca \ + --from-file=ca.crt= \ + -n api-forge-prod + ``` + +### Key Configuration Options + +```yaml +postgres: + # Image configuration + image: + repository: app_data_postgres_image + tag: latest + + # Replica count + replicas: 1 + + # Database names and users + database: appdb + username: appuser + + # Resource limits + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + + # Storage configuration + persistence: + data: + enabled: true + size: 20Gi + storageClass: "" # Use default + backups: + enabled: true + size: 40Gi + storageClass: "" + + # TLS configuration + tls: + enabled: true + existingSecret: postgres-tls + + # CA certificate (optional) + ca: + existingSecret: postgres-ca + + # Secrets reference + secrets: + existingSecret: postgres-secrets + + # High availability + podDisruptionBudget: + enabled: true + minAvailable: 1 + + # Network security (enable in production) + networkPolicy: + enabled: false + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: application + ports: + - protocol: TCP + port: 5432 +``` + +## Verification + +### Chart Validation +```bash +# Render templates to verify configuration +helm template test-postgres infra/helm/api-forge-bundled-postgres/ + +# Install with dry-run +helm install postgres infra/helm/api-forge-bundled-postgres/ --dry-run --debug +``` + +### Post-Deployment Checks +```bash +# Check pod status +kubectl get pods -n api-forge-prod -l app.kubernetes.io/name=postgres + +# Check StatefulSet +kubectl get statefulset postgres -n api-forge-prod + +# Check PVCs +kubectl get pvc -n api-forge-prod | grep postgres + +# Check service +kubectl get svc postgres -n api-forge-prod + +# Test database connection +kubectl exec -it postgres-0 -n api-forge-prod -- psql -U appuser -d appdb -c "SELECT version();" +``` + +## Migration from Main Chart + +If you previously deployed PostgreSQL as part of the main `api-forge` chart, follow these steps: + +1. **Backup your data**: + ```bash + kubectl exec -it postgres-0 -n api-forge-prod -- pg_dump -U appuser appdb > backup.sql + ``` + +2. **Uninstall main chart** (or upgrade with `postgres.enabled=false`) + +3. **Deploy standalone chart**: + ```bash + uv run api-forge-cli k8s db create + ``` + +4. **Restore data** (if needed) + +5. **Update main application** to connect to standalone PostgreSQL service + +## Comparison with Original + +| Feature | Original (Main Chart) | Standalone Chart | Status | +|---------|----------------------|------------------|--------| +| StatefulSet | ✅ | ✅ | ✅ Complete | +| Data PVC | ✅ | ✅ | ✅ Complete | +| Backup PVC | ✅ | ✅ | ✅ Complete | +| TLS/SSL | ✅ | ✅ | ✅ Complete | +| CA Certificates | ✅ | ✅ | ✅ Complete | +| Secret Validation | ✅ | ✅ | ✅ Complete | +| ConfigMap | ✅ | ✅ | ✅ Complete | +| Service | ✅ | ✅ | ✅ Complete | +| Network Policy | ✅ | ✅ | ✅ Complete | +| Pod Disruption Budget | ✅ | ✅ | ✅ Complete | +| 3-Role Security | ✅ | ✅ | ✅ Complete | +| Temporal Support | ✅ | ✅ | ✅ Complete | +| Health Probes | ✅ | ✅ | ✅ Complete | +| Resource Limits | ✅ | ✅ | ✅ Complete | + +## Known Limitations + +1. **Single Replica Default**: Chart defaults to 1 replica. For true HA, consider managed PostgreSQL services or external replication solutions. + +2. **No Automatic Backups**: Backup volume is provided, but backup scheduling must be configured separately. + +3. **No Built-in Replication**: For multi-replica setups, additional configuration for streaming replication is needed. + +4. **Manual Secret Creation**: Secrets must be created before chart deployment (not generated by chart). + +## Future Enhancements + +- [ ] Automated backup scheduling via CronJob +- [ ] Built-in replication support for HA +- [ ] Prometheus metrics ServiceMonitor +- [ ] Grafana dashboard ConfigMap +- [ ] pgBouncer integration for connection pooling +- [ ] Automated failover with Patroni/Stolon + +## Support + +For issues or questions about this standalone chart: +1. Check the main project documentation: `docs/postgres/` +2. Review deployment logs: `kubectl logs -n api-forge-prod postgres-0` +3. Open an issue in the project repository diff --git a/infra/helm/api-forge-bundled-postgres/files/.gitignore b/infra/helm/api-forge-bundled-postgres/files/.gitignore new file mode 100644 index 0000000..b6d2186 --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/files/.gitignore @@ -0,0 +1,10 @@ +# Generated files copied during deployment +# These are copied from the project root and should not be committed +.env +config.yaml +postgresql.conf +pg_hba.conf +verify-init.sh +01-init-app.sh +universal-entrypoint.sh +temporal/ diff --git a/infra/helm/api-forge-bundled-postgres/templates/configmap-env.yaml b/infra/helm/api-forge-bundled-postgres/templates/configmap-env.yaml new file mode 100644 index 0000000..878755c --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/templates/configmap-env.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.global.namePrefix }}-postgres-env + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + app.kubernetes.io/part-of: api-forge-bundled-postgres +data: + {{- /* Read environment variables from copied .env file */ -}} + {{- $envContent := .Files.Get "files/.env" }} + {{- if not $envContent }} + {{- fail "ERROR: files/.env not found. Run deployment to copy config files first." }} + {{- end }} + + {{- /* Load all variables from .env */ -}} + {{- range $line := splitList "\n" $envContent -}} + {{- if and (ne (trim $line) "") (not (hasPrefix "#" (trim $line))) -}} + {{- $parts := splitList "=" $line -}} + {{- if ge (len $parts) 2 -}} + {{- $key := index $parts 0 | trim -}} + {{- $rawValue := slice $parts 1 | join "=" | trim -}} + {{- /* Strip inline comments (everything after # with optional whitespace before it) */ -}} + {{- $value := regexReplaceAll "\\s*#.*$" $rawValue "" | trim }} + {{ $key }}: {{ $value | quote }} + {{- end -}} + {{- end -}} + {{- end }} + + # Postgres-specific entrypoint defaults (override .env if needed) + TZ: "UTC" + SECRETS_SOURCE_DIR: "/run/secrets" + SECRETS_TARGET_DIR: "/app/secrets" + CREATE_ENV_VARS: "true" + SKIP_USER_SWITCH: "false" + CERTS_TARGET_DIR: "/etc/postgresql/ssl" + CONTAINER_USER_UID: "postgres" + CONTAINER_USER_GID: "postgres" + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums" + PGDATA: "/var/lib/postgresql/data/pgdata" + POSTGRES_CONFIG_FILE: "/etc/postgresql/postgresql.conf" + POSTGRES_HBA_FILE: "/etc/postgresql/pg_hba.conf" + APP_SCHEMA: "app" diff --git a/infra/helm/api-forge/templates/configmaps/postgres-config.yaml b/infra/helm/api-forge-bundled-postgres/templates/configmap.yaml similarity index 79% rename from infra/helm/api-forge/templates/configmaps/postgres-config.yaml rename to infra/helm/api-forge-bundled-postgres/templates/configmap.yaml index 6aa2bdd..332af91 100644 --- a/infra/helm/api-forge/templates/configmaps/postgres-config.yaml +++ b/infra/helm/api-forge-bundled-postgres/templates/configmap.yaml @@ -1,11 +1,12 @@ -{{- if .Values.postgres.enabled }} apiVersion: v1 kind: ConfigMap metadata: name: {{ .Values.global.namePrefix }}-postgres-config namespace: {{ .Values.global.namespace }} labels: - {{- include "api-forge.labels" . | nindent 4 }} + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + app.kubernetes.io/part-of: api-forge-bundled-postgres data: postgresql.conf: | {{- .Files.Get "files/postgresql.conf" | nindent 4 }} @@ -20,4 +21,3 @@ data: verify-init.sh: | {{- .Files.Get "files/verify-init.sh" | nindent 4 }} -{{- end }} diff --git a/infra/helm/api-forge-bundled-postgres/templates/networkpolicy.yaml b/infra/helm/api-forge-bundled-postgres/templates/networkpolicy.yaml new file mode 100644 index 0000000..4ee6524 --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/templates/networkpolicy.yaml @@ -0,0 +1,31 @@ +{{- if .Values.postgres.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: postgres + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + app.kubernetes.io/part-of: api-forge-bundled-postgres +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: postgres + policyTypes: + - Ingress + ingress: + {{- range .Values.postgres.networkPolicy.ingress }} + - from: + {{- range .from }} + - podSelector: + matchLabels: + {{- toYaml .podSelector.matchLabels | nindent 10 }} + {{- end }} + ports: + {{- range .ports }} + - protocol: {{ .protocol }} + port: {{ .port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/infra/helm/api-forge-bundled-postgres/templates/poddisruptionbudget.yaml b/infra/helm/api-forge-bundled-postgres/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000..fc57fc0 --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/templates/poddisruptionbudget.yaml @@ -0,0 +1,16 @@ +{{- if .Values.postgres.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: postgres + namespace: {{ .Values.global.namespace }} + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + app.kubernetes.io/part-of: api-forge-bundled-postgres +spec: + minAvailable: {{ .Values.postgres.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + app.kubernetes.io/name: postgres +{{- end }} diff --git a/infra/helm/api-forge/templates/storage/postgres-backups-pvc.yaml b/infra/helm/api-forge-bundled-postgres/templates/pvc-backups.yaml similarity index 64% rename from infra/helm/api-forge/templates/storage/postgres-backups-pvc.yaml rename to infra/helm/api-forge-bundled-postgres/templates/pvc-backups.yaml index 8a2c839..507b517 100644 --- a/infra/helm/api-forge/templates/storage/postgres-backups-pvc.yaml +++ b/infra/helm/api-forge-bundled-postgres/templates/pvc-backups.yaml @@ -1,4 +1,3 @@ -{{- if .Values.postgres.enabled }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -6,15 +5,14 @@ metadata: namespace: {{ .Values.global.namespace }} labels: app.kubernetes.io/name: postgres - app.kubernetes.io/component: database-backup - app.kubernetes.io/part-of: api-forge + app.kubernetes.io/component: database + app.kubernetes.io/part-of: api-forge-bundled-postgres spec: accessModes: - ReadWriteOnce resources: requests: - storage: {{ .Values.postgres.persistence.backups.size | default "50Gi" }} + storage: {{ .Values.postgres.persistence.backups.size }} {{- if .Values.postgres.persistence.backups.storageClass }} storageClassName: {{ .Values.postgres.persistence.backups.storageClass }} {{- end }} -{{- end }} diff --git a/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-ca.yaml b/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-ca.yaml new file mode 100644 index 0000000..6fe14d0 --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-ca.yaml @@ -0,0 +1,3 @@ +{{- if not .Values.postgres.ca.existingSecret }} +{{- fail "postgres.ca.existingSecret must be provided" }} +{{- end }} diff --git a/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-secrets.yaml b/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-secrets.yaml new file mode 100644 index 0000000..b392beb --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-secrets.yaml @@ -0,0 +1,3 @@ +{{- if not .Values.postgres.secrets.existingSecret }} +{{- fail "postgres.secrets.existingSecret must be provided" }} +{{- end }} diff --git a/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-tls.yaml b/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-tls.yaml new file mode 100644 index 0000000..b2565dd --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/templates/secrets/postgres-tls.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.postgres.tls.enabled (not .Values.postgres.tls.existingSecret) }} +{{- fail "postgres.tls.existingSecret must be provided when TLS is enabled" }} +{{- end }} diff --git a/infra/helm/api-forge/templates/services/postgres-service.yaml b/infra/helm/api-forge-bundled-postgres/templates/service.yaml similarity index 81% rename from infra/helm/api-forge/templates/services/postgres-service.yaml rename to infra/helm/api-forge-bundled-postgres/templates/service.yaml index 0afcea7..d8fe928 100644 --- a/infra/helm/api-forge/templates/services/postgres-service.yaml +++ b/infra/helm/api-forge-bundled-postgres/templates/service.yaml @@ -1,4 +1,3 @@ -{{- if .Values.postgres.enabled }} apiVersion: v1 kind: Service metadata: @@ -7,7 +6,7 @@ metadata: labels: app.kubernetes.io/name: postgres app.kubernetes.io/component: database - app.kubernetes.io/part-of: api-forge + app.kubernetes.io/part-of: api-forge-bundled-postgres spec: type: ClusterIP ports: @@ -18,4 +17,3 @@ spec: selector: app.kubernetes.io/name: postgres sessionAffinity: None -{{- end }} diff --git a/infra/helm/api-forge/templates/deployments/postgres.yaml b/infra/helm/api-forge-bundled-postgres/templates/statefulset.yaml similarity index 66% rename from infra/helm/api-forge/templates/deployments/postgres.yaml rename to infra/helm/api-forge-bundled-postgres/templates/statefulset.yaml index efc4a86..a2a6b8f 100644 --- a/infra/helm/api-forge/templates/deployments/postgres.yaml +++ b/infra/helm/api-forge-bundled-postgres/templates/statefulset.yaml @@ -1,70 +1,47 @@ -{{- if .Values.postgres.enabled }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: postgres + name: {{ .Values.global.namePrefix }}-postgres namespace: {{ .Values.global.namespace }} labels: app.kubernetes.io/name: postgres + app.kubernetes.io/instance: {{ .Values.global.namePrefix }}-postgres app.kubernetes.io/component: database - app.kubernetes.io/part-of: api-forge + app.kubernetes.io/part-of: api-forge-bundled-postgres app.kubernetes.io/version: "15" spec: - replicas: 1 - strategy: - type: Recreate # Required for StatefulSet-like behavior with PVCs + serviceName: postgres + replicas: {{ .Values.postgres.replicas }} selector: matchLabels: app.kubernetes.io/name: postgres + updateStrategy: + type: RollingUpdate + podManagementPolicy: OrderedReady template: metadata: labels: app.kubernetes.io/name: postgres + app.kubernetes.io/instance: {{ .Values.global.namePrefix }}-postgres app.kubernetes.io/component: database - app.kubernetes.io/part-of: api-forge + app.kubernetes.io/part-of: api-forge-bundled-postgres spec: securityContext: - # Container must start as root to run entrypoint script - # Script will drop privileges to user 70 (postgres) via gosu fsGroup: 70 seccompProfile: type: RuntimeDefault containers: - name: postgres - image: {{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag | default "latest" }} - imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }} + image: {{ .Values.postgres.image.repository }}:{{ .Values.postgres.image.tag }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} command: ["/opt/entry/start-scripts/universal-entrypoint.sh"] args: ["/bin/sh", "-lc", "/opt/entry/start-scripts/pg-start.sh"] - # Load all app environment variables from ConfigMap + # Load all postgres environment variables from ConfigMap envFrom: - configMapRef: - name: {{ .Values.global.namePrefix }}-app-env + name: {{ .Values.global.namePrefix }}-postgres-env env: - # Override/add specific values - - name: TZ - value: UTC - - name: SECRETS_SOURCE_DIR - value: /run/secrets - - name: SECRETS_TARGET_DIR - value: /app/secrets - - name: CREATE_ENV_VARS - value: "true" - - name: SKIP_USER_SWITCH - value: "false" - - name: CERTS_TARGET_DIR - value: /etc/postgresql/ssl - - name: CONTAINER_USER_UID - value: postgres - - name: CONTAINER_USER_GID - value: postgres - - name: POSTGRES_INITDB_ARGS - value: "--auth-host=scram-sha-256 --data-checksums" - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - - name: POSTGRES_CONFIG_FILE - value: /etc/postgresql/postgresql.conf - - name: POSTGRES_HBA_FILE - value: /etc/postgresql/pg_hba.conf + # Override/add specific values here if needed ports: - name: postgresql containerPort: 5432 @@ -90,14 +67,9 @@ spec: timeoutSeconds: 5 failureThreshold: 20 resources: - requests: - cpu: 500m - memory: 512Mi - limits: - cpu: 2000m - memory: 2Gi + {{- toYaml .Values.postgres.resources | nindent 12 }} volumeMounts: - # Secrets - each mounted as individual file + # Secrets - mounted as individual files - name: postgres-secrets mountPath: /run/secrets/postgres_password subPath: postgres_password @@ -119,6 +91,7 @@ spec: subPath: postgres_temporal_pw readOnly: true # TLS certificates + {{- if .Values.postgres.tls.enabled }} - name: postgres-tls mountPath: /run/secrets/server.crt subPath: server.crt @@ -127,6 +100,7 @@ spec: mountPath: /run/secrets/server.key subPath: server.key readOnly: true + {{- end }} # Config files - name: postgres-config mountPath: /opt/entry/init-scripts @@ -139,9 +113,10 @@ spec: mountPath: /etc/postgresql/pg_hba.conf subPath: pg_hba.conf readOnly: true - # Data volumes + # Data volume - name: postgres-data mountPath: /var/lib/postgresql/data + # Backups volume - name: postgres-backups mountPath: /var/lib/postgresql/backups # Shared memory @@ -150,19 +125,18 @@ spec: volumes: - name: postgres-secrets secret: - secretName: postgres-secrets + secretName: {{ .Values.postgres.secrets.existingSecret }} defaultMode: 0400 + {{- if .Values.postgres.tls.enabled }} - name: postgres-tls secret: - secretName: postgres-tls + secretName: {{ .Values.postgres.tls.existingSecret }} defaultMode: 0400 + {{- end }} - name: postgres-config configMap: name: {{ .Values.global.namePrefix }}-postgres-config defaultMode: 0755 - - name: postgres-data - persistentVolumeClaim: - claimName: postgres-data - name: postgres-backups persistentVolumeClaim: claimName: postgres-backups @@ -170,5 +144,17 @@ spec: emptyDir: medium: Memory sizeLimit: 1Gi - restartPolicy: Always -{{- end }} + volumeClaimTemplates: + - metadata: + name: postgres-data + labels: + app.kubernetes.io/name: postgres + app.kubernetes.io/component: database + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.persistence.data.size }} + {{- if .Values.postgres.persistence.data.storageClass }} + storageClassName: {{ .Values.postgres.persistence.data.storageClass }} + {{- end }} diff --git a/infra/helm/api-forge-bundled-postgres/values.yaml b/infra/helm/api-forge-bundled-postgres/values.yaml new file mode 100644 index 0000000..6607f3b --- /dev/null +++ b/infra/helm/api-forge-bundled-postgres/values.yaml @@ -0,0 +1,81 @@ +# ============================================================================= +# API Forge Bundled PostgreSQL Helm Chart Values +# ============================================================================= +# This is a standalone PostgreSQL chart extracted from the main api-forge chart. +# It can be deployed independently for users who want to manage the database +# separately from the application. +# +# For managed PostgreSQL services (AWS RDS, Google Cloud SQL, etc.), skip this +# chart and configure your application to connect to the external database. +# ============================================================================= + +global: + namespace: api-forge-prod + namePrefix: api-forge + imagePullPolicy: IfNotPresent + +postgres: + image: + repository: app_data_postgres_image + tag: latest + + replicas: 1 + + database: appdb + username: appuser + + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + + # Persistence configuration + persistence: + data: + enabled: true + size: 20Gi + storageClass: "" # Use default storage class + backups: + enabled: true + size: 40Gi + storageClass: "" + + # PostgreSQL configuration + config: + maxConnections: 200 + sharedBuffers: 256MB + effectiveCacheSize: 1GB + + # TLS Configuration + tls: + enabled: true + existingSecret: postgres-tls + + # CA Certificate for client verification (optional) + ca: + existingSecret: postgres-ca + + # Database secrets + secrets: + existingSecret: postgres-secrets + + # Pod Disruption Budget + podDisruptionBudget: + enabled: true + minAvailable: 1 + + # Network Policy - restricts database access to authorized pods only + # Set enabled: true in production for better security + networkPolicy: + enabled: false + ingress: + - from: + - podSelector: + matchLabels: + app.kubernetes.io/component: application + ports: + - protocol: TCP + port: 5432 diff --git a/infra/helm/api-forge/templates/configmaps/app-env.yaml b/infra/helm/api-forge/templates/configmaps/app-env.yaml index 4fa5932..d9cab50 100644 --- a/infra/helm/api-forge/templates/configmaps/app-env.yaml +++ b/infra/helm/api-forge/templates/configmaps/app-env.yaml @@ -19,7 +19,9 @@ data: {{- $parts := splitList "=" $line -}} {{- if ge (len $parts) 2 -}} {{- $key := index $parts 0 | trim -}} - {{- $value := slice $parts 1 | join "=" | trim -}} + {{- $rawValue := slice $parts 1 | join "=" | trim -}} + {{- /* Strip inline comments (everything after # with optional whitespace before it) */ -}} + {{- $value := regexReplaceAll "\\s*#.*$" $rawValue "" | trim -}} {{- if ne $key "APP_ENVIRONMENT" }} {{ $key }}: {{ $value | quote }} {{- end -}} diff --git a/infra/helm/api-forge/templates/jobs/postgres-verifier.yaml b/infra/helm/api-forge/templates/jobs/postgres-verifier.yaml index b035793..c8f5e08 100644 --- a/infra/helm/api-forge/templates/jobs/postgres-verifier.yaml +++ b/infra/helm/api-forge/templates/jobs/postgres-verifier.yaml @@ -1,4 +1,14 @@ -{{- if .Values.postgres.enabled }} +{{/* +NOTE: postgres-verifier has been replaced by the CLI `db verify` command. +Run `uv run api-forge-cli db verify` to verify PostgreSQL setup before deployment. + +The pre-deployment database check is now enforced by the `prod up` and `k8s up` commands, +which fail if the database is not accessible (unless --skip-db-check is used). + +This template is kept for reference but is disabled by default. +To re-enable, change the condition below to: if .Values.postgres.verifier.enabled +*/}} +{{- if false }} apiVersion: batch/v1 kind: Job metadata: diff --git a/infra/helm/api-forge/templates/jobs/temporal-namespace-init.yaml b/infra/helm/api-forge/templates/jobs/temporal-namespace-init.yaml index 0e23579..d5853ae 100644 --- a/infra/helm/api-forge/templates/jobs/temporal-namespace-init.yaml +++ b/infra/helm/api-forge/templates/jobs/temporal-namespace-init.yaml @@ -1,4 +1,4 @@ -{{- if .Values.temporal.enabled }} +{{- if and .Values.temporal.enabled .Values.temporal.jobs.namespaceInit.enabled }} apiVersion: batch/v1 kind: Job metadata: @@ -9,9 +9,13 @@ metadata: app.kubernetes.io/component: workflow-engine-init app.kubernetes.io/part-of: api-forge annotations: - # Run on install and upgrade, delete previous job before creating new one - # Weight 15 ensures this runs after schema-setup (weight 5) and verifier (weight 10) + # Run on install only (namespace creation is idempotent but only needed once) + # To re-run namespace init, set temporal.jobs.namespaceInit.runOnUpgrade=true + {{- if .Values.temporal.jobs.namespaceInit.runOnUpgrade }} "helm.sh/hook": post-install,post-upgrade + {{- else }} + "helm.sh/hook": post-install + {{- end }} "helm.sh/hook-weight": "15" "helm.sh/hook-delete-policy": before-hook-creation spec: @@ -43,10 +47,8 @@ spec: echo "Waiting for Temporal server to be ready..." until nc -z -w 2 temporal 7233; do echo "Temporal is unavailable - sleeping" - sleep 5 + sleep 2 done - echo "Temporal is up - waiting 10 more seconds for full initialization" - sleep 10 echo "Temporal is ready!" securityContext: allowPrivilegeEscalation: false diff --git a/infra/helm/api-forge/templates/jobs/temporal-schema-setup.yaml b/infra/helm/api-forge/templates/jobs/temporal-schema-setup.yaml index 65efdbf..560f824 100644 --- a/infra/helm/api-forge/templates/jobs/temporal-schema-setup.yaml +++ b/infra/helm/api-forge/templates/jobs/temporal-schema-setup.yaml @@ -1,4 +1,4 @@ -{{- if .Values.temporal.enabled }} +{{- if and .Values.temporal.enabled .Values.temporal.jobs.schemaSetup.enabled }} apiVersion: batch/v1 kind: Job metadata: @@ -9,8 +9,13 @@ metadata: app.kubernetes.io/component: workflow-engine-setup app.kubernetes.io/part-of: api-forge annotations: - # Run on install and upgrade, delete previous job before creating new one + # Run on install only (schema setup is idempotent but only needed once) + # To re-run schema setup, set temporal.jobs.schemaSetup.runOnUpgrade=true + {{- if .Values.temporal.jobs.schemaSetup.runOnUpgrade }} "helm.sh/hook": post-install,post-upgrade + {{- else }} + "helm.sh/hook": post-install + {{- end }} "helm.sh/hook-weight": "5" "helm.sh/hook-delete-policy": before-hook-creation spec: @@ -44,8 +49,6 @@ spec: echo "PostgreSQL is unavailable - sleeping" sleep 2 done - echo "PostgreSQL is up - waiting 5 more seconds for full initialization" - sleep 5 echo "PostgreSQL is ready!" securityContext: allowPrivilegeEscalation: false diff --git a/infra/helm/api-forge/templates/network-policies/postgres-netpol.yaml b/infra/helm/api-forge/templates/network-policies/postgres-netpol.yaml deleted file mode 100644 index b8010a6..0000000 --- a/infra/helm/api-forge/templates/network-policies/postgres-netpol.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if and .Values.postgres.enabled .Values.postgres.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: postgres-ingress - namespace: {{ .Values.global.namespace }} - labels: - {{- include "api-forge.labels" . | nindent 4 }} -spec: - podSelector: - matchLabels: - app.kubernetes.io/component: database - policyTypes: - - Ingress - ingress: - {{- toYaml .Values.postgres.networkPolicy.ingress | nindent 4 }} -{{- end }} diff --git a/infra/helm/api-forge/templates/poddisruptionbudgets/postgres-pdb.yaml b/infra/helm/api-forge/templates/poddisruptionbudgets/postgres-pdb.yaml deleted file mode 100644 index 0664ddf..0000000 --- a/infra/helm/api-forge/templates/poddisruptionbudgets/postgres-pdb.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if and .Values.postgres.enabled .Values.postgres.podDisruptionBudget.enabled }} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: postgres - namespace: {{ .Values.global.namespace }} - labels: - app.kubernetes.io/name: postgres - app.kubernetes.io/component: database - app.kubernetes.io/part-of: api-forge -spec: - {{- if .Values.postgres.podDisruptionBudget.minAvailable }} - minAvailable: {{ .Values.postgres.podDisruptionBudget.minAvailable }} - {{- else if .Values.postgres.podDisruptionBudget.maxUnavailable }} - maxUnavailable: {{ .Values.postgres.podDisruptionBudget.maxUnavailable }} - {{- else }} - # Default: PostgreSQL must always be available (single replica setup) - minAvailable: 1 - {{- end }} - selector: - matchLabels: - app.kubernetes.io/name: postgres -{{- end }} diff --git a/infra/helm/api-forge/templates/secrets/postgres-ca.yaml b/infra/helm/api-forge/templates/secrets/postgres-ca.yaml index eaecfdd..d04588a 100644 --- a/infra/helm/api-forge/templates/secrets/postgres-ca.yaml +++ b/infra/helm/api-forge/templates/secrets/postgres-ca.yaml @@ -1,3 +1,7 @@ +{{- if .Values.postgres }} +{{- if .Values.postgres.enabled }} {{- if not .Values.postgres.ca.existingSecret }} {{- fail "postgresql.ca.existingSecret must be provided" }} {{- end }} +{{- end }} +{{- end }} diff --git a/infra/helm/api-forge/templates/secrets/postgres-secrets.yaml b/infra/helm/api-forge/templates/secrets/postgres-secrets.yaml index b3820b3..de76880 100644 --- a/infra/helm/api-forge/templates/secrets/postgres-secrets.yaml +++ b/infra/helm/api-forge/templates/secrets/postgres-secrets.yaml @@ -1,3 +1,7 @@ +{{- if .Values.postgres }} +{{- if .Values.postgres.enabled }} {{- if not .Values.postgres.secrets.existingSecret }} {{- fail "postgresql.secrets.existingSecret must be set when PostgreSQL is enabled" }} {{- end }} +{{- end }} +{{- end }} diff --git a/infra/helm/api-forge/templates/storage/postgres-data-pvc.yaml b/infra/helm/api-forge/templates/storage/postgres-data-pvc.yaml deleted file mode 100644 index b8c0315..0000000 --- a/infra/helm/api-forge/templates/storage/postgres-data-pvc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.postgres.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgres-data - namespace: {{ .Values.global.namespace }} - labels: - app.kubernetes.io/name: postgres - app.kubernetes.io/component: database - app.kubernetes.io/part-of: api-forge -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.postgres.persistence.data.size | default "20Gi" }} - {{- if .Values.postgres.persistence.data.storageClass }} - storageClassName: {{ .Values.postgres.persistence.data.storageClass }} - {{- end }} -{{- end }} diff --git a/infra/helm/api-forge/values-bitnami-postgres.yaml b/infra/helm/api-forge/values-bitnami-postgres.yaml new file mode 100644 index 0000000..20d45a2 --- /dev/null +++ b/infra/helm/api-forge/values-bitnami-postgres.yaml @@ -0,0 +1,54 @@ +# Bitnami PostgreSQL Helm Chart Values Override +# This file customizes the Bitnami PostgreSQL chart deployment +# to match the api-forge application requirements + +# Override service and resource names to match .env configuration +fullnameOverride: "postgres" +nameOverride: "postgres" + +# Primary PostgreSQL configuration +primary: + service: + nameOverride: "postgres" + + # Resource limits + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2000m + memory: 4Gi + + # Persistence + persistence: + enabled: true + size: 20Gi + +# Authentication +auth: + enablePostgresUser: true + # Use existing secret for postgres superuser password + existingSecret: postgres-secrets + secretKeys: + adminPasswordKey: postgres_password + # Note: Additional app users (appuser, appowner, etc.) are created via k8s db init + # username: appuser + # password: + # database: appdb + +# TLS/SSL Configuration +tls: + enabled: true + certificatesSecret: postgres-tls + certFilename: server.crt + certKeyFilename: server.key + +# Metrics (optional) +metrics: + enabled: false + +# Pod disruption budget +pdb: + create: true + minAvailable: 1 diff --git a/infra/helm/api-forge/values.yaml b/infra/helm/api-forge/values.yaml index 9621158..d6bed43 100644 --- a/infra/helm/api-forge/values.yaml +++ b/infra/helm/api-forge/values.yaml @@ -24,57 +24,27 @@ global: # ============================================================================= # PostgreSQL Database # ============================================================================= -# For production, consider using a managed PostgreSQL service (RDS, Cloud SQL) -# instead of running PostgreSQL in-cluster for better HA and backup support. +# PostgreSQL is now managed via a separate standalone chart: +# infra/helm/api-forge-bundled-postgres/ +# +# Deploy PostgreSQL separately using: +# uv run api-forge-cli k8s db create +# +# Or use a managed database service (AWS RDS, Google Cloud SQL, etc.) and +# configure your application to connect to it via DATABASE_URL environment variable. +# ============================================================================= postgres: + # enabled: false means postgres is NOT deployed by this chart + # The app still needs database credentials via secrets enabled: true - image: - repository: app_data_postgres_image - tag: latest - replicas: 1 - database: appdb - username: appuser - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 2000m - memory: 4Gi - persistence: - data: - enabled: true - size: 20Gi - storageClass: '' - backups: - enabled: true - size: 40Gi - storageClass: '' - config: - maxConnections: 200 - sharedBuffers: 256MB - effectiveCacheSize: 1GB + secrets: + # Reference to existing secret containing postgres credentials + # Created by: infra/helm/api-forge/scripts/apply-secrets.sh + existingSecret: postgres-secrets tls: - enabled: true existingSecret: postgres-tls ca: existingSecret: postgres-ca - secrets: - existingSecret: postgres-secrets - # PodDisruptionBudget - ensures database availability during maintenance - podDisruptionBudget: - enabled: true - minAvailable: 1 - networkPolicy: - enabled: true - ingress: - - from: - - podSelector: - matchLabels: - app.kubernetes.io/component: application - ports: - - protocol: TCP - port: 5432 # ============================================================================= # Redis Cache/Sessions @@ -156,11 +126,15 @@ temporal: jobs: schemaSetup: enabled: true + # Set to true to run schema setup on every upgrade (default: only on install) + runOnUpgrade: false image: temporalio/admin-tools:latest backoffLimit: 3 ttlSecondsAfterFinished: 300 namespaceInit: enabled: true + # Set to true to run namespace init on every upgrade (default: only on install) + runOnUpgrade: false image: temporalio/admin-tools:latest backoffLimit: 3 ttlSecondsAfterFinished: 300 @@ -295,10 +269,10 @@ app: # nginx.ingress.kubernetes.io/proxy-body-size: "10m" # cert-manager.io/cluster-issuer: letsencrypt-prod hosts: - - host: api-forge.local - paths: - - path: / - pathType: Prefix + - host: api-forge.local + paths: + - path: / + pathType: Prefix tls: [] # - secretName: api-forge-tls # hosts: diff --git a/infra/scripts/entrypoint.sh b/infra/scripts/entrypoint.sh deleted file mode 100644 index 8516ba1..0000000 --- a/infra/scripts/entrypoint.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env sh -set -eu - -# Application user and group IDs (must match Dockerfile) -APP_UID=1001 -APP_GID=1001 - -echo "Starting entrypoint script..." - -# Create secure tmpfs directory for secrets with proper ownership -echo "Setting up secure secrets directory..." -install -d -m 0700 -o "$APP_UID" -g "$APP_GID" /run/secrets - -# Copy all mounted host secrets to tmpfs with proper ownership and permissions -echo "Copying secrets with proper permissions..." -for secret_file in /mnt/host_secrets/*; do - if [ -f "$secret_file" ]; then - secret_name=$(basename "$secret_file") - # Remove .txt extension if present to match config expectations - target_name="${secret_name%.txt}" - echo "Installing secret: $secret_name -> $target_name" - install -m 0400 -o "$APP_UID" -g "$APP_GID" "$secret_file" "/run/secrets/$target_name" - fi -done - -echo "Secrets setup complete. Starting application as appuser..." - -# Drop privileges and start the application -exec su-exec "$APP_UID:$APP_GID" "$@" \ No newline at end of file diff --git a/infra/secrets/generate_secrets.sh b/infra/secrets/generate_secrets.sh index af6dbbc..6ee5c7d 100755 --- a/infra/secrets/generate_secrets.sh +++ b/infra/secrets/generate_secrets.sh @@ -337,11 +337,21 @@ create_ca_bundle() { local ca_bundle="$CERTS_DIR/ca-bundle.crt" local int_crt="$CERTS_DIR/intermediate-ca.crt" local root_crt="$CERTS_DIR/root-ca.crt" + local external_pg_ca="$CERTS_DIR/ca-bundle-postgres-external.crt" print_info "Creating CA bundle for client certificate validation..." # Create CA bundle (intermediate CA + root CA) for ssl_ca_file cat "$int_crt" "$root_crt" > "$ca_bundle" + + # Include external PostgreSQL CA if it exists (for Aiven, RDS, etc.) + if [ -f "$external_pg_ca" ]; then + print_info "Including external PostgreSQL CA certificate..." + echo "" >> "$ca_bundle" + echo "# External PostgreSQL CA Certificate" >> "$ca_bundle" + cat "$external_pg_ca" >> "$ca_bundle" + fi + chmod 644 "$ca_bundle" print_success "Created CA bundle: certs/ca-bundle.crt" print_info "Use this file for PostgreSQL ssl_ca_file parameter" diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 0000000..8d8042e --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,229 @@ +# Database Migrations + +This directory contains Alembic database migration scripts for managing schema changes. + +## Overview + +- **Tool**: Alembic (SQLAlchemy migration tool) +- **Models**: SQLModel (based on SQLAlchemy) +- **Configuration**: `alembic.ini` (project root) +- **Environment**: `migrations/env.py` +- **Migration Scripts**: `migrations/versions/` + +## Usage + +All migration commands are integrated into the CLI and automatically handle port-forwarding for Kubernetes deployments. + +### Apply Migrations + +```bash +# Apply all pending migrations +uv run api-forge-cli k8s db migrate upgrade + +# Apply up to a specific revision +uv run api-forge-cli k8s db migrate upgrade abc123 +``` + +### Create New Migration + +```bash +# Auto-generate migration from model changes (recommended) +uv run api-forge-cli k8s db migrate revision "add user table" + +# Create empty migration template for manual changes +uv run api-forge-cli k8s db migrate revision "custom changes" --no-autogenerate +``` + +### Rollback Migrations + +```bash +# Rollback to a specific revision +uv run api-forge-cli k8s db migrate downgrade abc123 + +# Rollback one migration +uv run api-forge-cli k8s db migrate downgrade -1 + +# Rollback all migrations (to base) +uv run api-forge-cli k8s db migrate downgrade base +``` + +### View Migration State + +```bash +# Show current migration revision +uv run api-forge-cli k8s db migrate current + +# Show full migration history +uv run api-forge-cli k8s db migrate history +``` + +### Generate SQL (Dry Run) + +```bash +# See what SQL would be executed without running it +uv run api-forge-cli k8s db migrate upgrade --sql +uv run api-forge-cli k8s db migrate downgrade abc123 --sql +``` + +## Workflow + +### 1. Make Model Changes + +Edit your SQLModel models in `src/app/entities/`: + +```python +# src/app/entities/user/models.py +from sqlmodel import Field, SQLModel + +class User(SQLModel, table=True): + id: int = Field(primary_key=True) + email: str = Field(unique=True, index=True) + name: str + # Add new field: + is_active: bool = Field(default=True) +``` + +### 2. Import Models in env.py + +Ensure your models are imported in `migrations/env.py` so Alembic can detect them: + +```python +# In migrations/env.py, add: +from src.app.entities.user.models import User +from src.app.entities.book.models import Book +# ... import all your models +``` + +### 3. Generate Migration + +```bash +uv run api-forge-cli k8s db migrate revision "add user is_active field" +``` + +This will: +- Compare your models to the current database schema +- Generate a migration file in `migrations/versions/` +- Include upgrade() and downgrade() functions + +### 4. Review Migration + +Check the generated file in `migrations/versions/YYYYMMDD_HHMM_slug.py`: + +```python +def upgrade() -> None: + op.add_column('user', sa.Column('is_active', sa.Boolean(), nullable=False, server_default='true')) + +def downgrade() -> None: + op.drop_column('user', 'is_active') +``` + +Edit if needed (add data migrations, custom logic, etc.) + +### 5. Apply Migration + +```bash +uv run api-forge-cli k8s db migrate upgrade +``` + +### 6. Verify + +```bash +uv run api-forge-cli k8s db migrate current +uv run api-forge-cli k8s db verify +``` + +## Best Practices + +1. **Always review auto-generated migrations** - Alembic might not perfectly detect all changes +2. **Test migrations locally first** - Use a dev environment before production +3. **Keep migrations small and focused** - One logical change per migration +4. **Never edit applied migrations** - Create new migrations for changes +5. **Use meaningful names** - Describe what the migration does +6. **Handle data migrations carefully** - Consider downtime and large datasets +7. **Backup before major migrations** - Use `k8s db backup` first + +## Integration with Kubernetes + +The CLI automatically: +- Establishes port-forwarding to the PostgreSQL pod +- Loads database credentials from secrets +- Handles both bundled and external databases +- Works in any namespace/environment + +## Troubleshooting + +### "Target database is not up to date" + +```bash +# Check current state +uv run api-forge-cli k8s db migrate current + +# Apply pending migrations +uv run api-forge-cli k8s db migrate upgrade +``` + +### "Can't locate revision identified by 'abc123'" + +The revision doesn't exist. Check available revisions: + +```bash +uv run api-forge-cli k8s db migrate history +``` + +### Auto-generation not detecting changes + +Ensure your models are imported in `migrations/env.py`: + +```python +# Add at top of migrations/env.py +from src.app.entities.user.models import User +from src.app.entities.book.models import Book +# ... all models that should be tracked +``` + +### Migration conflicts + +If multiple developers create migrations simultaneously: + +```bash +# Merge migration branches +alembic merge -m "merge branches" head1 head2 + +# Or manually edit migration to depend on both +``` + +## Migration File Structure + +``` +migrations/ +├── env.py # Alembic environment configuration +├── script.py.mako # Template for new migrations +└── versions/ # Migration scripts + ├── 20231201_1430_initial_schema.py + ├── 20231202_0900_add_user_table.py + └── 20231203_1600_add_indexes.py +``` + +## Production Deployment + +Include migration in your deployment pipeline: + +```bash +# 1. Backup database +uv run api-forge-cli k8s db backup + +# 2. Apply migrations +uv run api-forge-cli k8s db migrate upgrade + +# 3. Verify +uv run api-forge-cli k8s db verify + +# 4. Deploy application +uv run api-forge-cli deploy up k8s +``` + +## Further Reading + +- [Alembic Documentation](https://alembic.sqlalchemy.org/) +- [SQLModel Documentation](https://sqlmodel.tiangolo.com/) +- [Migration Best Practices](https://alembic.sqlalchemy.org/en/latest/tutorial.html) diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..6c02252 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,98 @@ +"""Alembic environment configuration for database migrations. + +This module configures the Alembic migration environment to work with: +- SQLModel/SQLAlchemy models +- Both online (CLI) and offline (SQL file generation) modes +- Kubernetes port-forwarding for database access (handled by CLI) +- Environment-specific database URLs +""" + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# SQLModel uses SQLModel.metadata as its declarative base metadata. +# All models with `table=True` automatically register there when imported. +# The loader dynamically discovers and imports all table.py modules. +from src.app.entities.loader import get_metadata + +# target_metadata is what Alembic uses for autogeneration +# get_metadata() imports all tables and returns SQLModel.metadata +target_metadata = get_metadata() + +# This is the Alembic Config object +config = context.config + +# Interpret the config file for Python logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Get database URL from environment variable (set by CLI) +# The CLI handles port-forwarding and constructs the correct URL +database_url = os.environ.get("DATABASE_URL") + +if not database_url: + raise RuntimeError( + "DATABASE_URL environment variable not set. " + "Run migrations via the CLI: uv run api-forge-cli k8s db migrate" + ) + +# Override sqlalchemy.url in alembic.ini with runtime value +config.set_main_option("sqlalchemy.url", database_url) + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + compare_server_default=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + compare_server_default=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..3124b62 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/20251224_0327_initial_schema.py b/migrations/versions/20251224_0327_initial_schema.py new file mode 100644 index 0000000..0c3fc90 --- /dev/null +++ b/migrations/versions/20251224_0327_initial_schema.py @@ -0,0 +1,112 @@ +"""initial schema + +Revision ID: 19becf30b774 +Revises: +Create Date: 2025-12-24 03:27:53.544051+00:00 + +""" + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +# revision identifiers, used by Alembic. +revision = "19becf30b774" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "booktable", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column( + "updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "producttable", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column( + "updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "usertable", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column( + "updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False + ), + sa.Column("first_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("last_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("email", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("phone", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("address", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "useridentitytable", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column( + "updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False + ), + sa.Column("issuer", sa.String(length=512), nullable=False), + sa.Column("subject", sa.String(length=512), nullable=False), + sa.Column("uid_claim", sa.String(length=512), nullable=True), + sa.Column("user_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["usertable.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("issuer", "subject", name="uq_identity_issuer_subject"), + ) + op.create_index( + op.f("ix_useridentitytable_issuer"), + "useridentitytable", + ["issuer"], + unique=False, + ) + op.create_index( + op.f("ix_useridentitytable_subject"), + "useridentitytable", + ["subject"], + unique=False, + ) + op.create_index( + op.f("ix_useridentitytable_uid_claim"), + "useridentitytable", + ["uid_claim"], + unique=False, + ) + op.create_index( + op.f("ix_useridentitytable_user_id"), + "useridentitytable", + ["user_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_useridentitytable_user_id"), table_name="useridentitytable") + op.drop_index( + op.f("ix_useridentitytable_uid_claim"), table_name="useridentitytable" + ) + op.drop_index(op.f("ix_useridentitytable_subject"), table_name="useridentitytable") + op.drop_index(op.f("ix_useridentitytable_issuer"), table_name="useridentitytable") + op.drop_table("useridentitytable") + op.drop_table("usertable") + op.drop_table("producttable") + op.drop_table("booktable") + # ### end Alembic commands ### diff --git a/pyproject.toml b/pyproject.toml index a802980..b4fa524 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ dependencies = [ "requests>=2.32.5", "ruamel.yaml>=0.18.6", "kr8s>=0.20.14", + "alembic>=1.13.0", + "click>=8.2.1", ] [build-system] @@ -50,6 +52,7 @@ dev = [ "types-authlib>=1.6.2.20250914", "copier>=9.0.0", "pre-commit>=4.5.0", + "types-psycopg2>=2.9.21.20251012", ] [project.scripts] @@ -91,6 +94,7 @@ warn_redundant_casts = true warn_unused_ignores = true warn_return_any = true strict_optional = true +disable_error_code = ["no-any-return"] [[tool.mypy.overrides]] module = "tests.*" diff --git a/src/app/core/services/database/db_manage.py b/src/app/core/services/database/db_manage.py index 2fb8a50..ed742c3 100644 --- a/src/app/core/services/database/db_manage.py +++ b/src/app/core/services/database/db_manage.py @@ -3,6 +3,7 @@ from loguru import logger from sqlmodel import create_engine +from src.app.entities.loader import get_metadata from src.app.runtime.context import get_config main_config = get_config() @@ -17,8 +18,7 @@ def create_all(self) -> None: """Create all database tables.""" from sqlmodel import SQLModel - from src.app.entities.core.user import UserTable # noqa: F401 - from src.app.entities.core.user_identity import UserIdentityTable # noqa: F401 + get_metadata() # Ensure all tables are imported and registered SQLModel.metadata.create_all(self._engine) logger.info("Database initialized with tables.") diff --git a/src/app/entities/__init__.py b/src/app/entities/__init__.py index 2de20b2..401f329 100644 --- a/src/app/entities/__init__.py +++ b/src/app/entities/__init__.py @@ -15,6 +15,8 @@ from .core.user import User, UserRepository, UserTable from .core.user_identity import UserIdentity, UserIdentityRepository, UserIdentityTable +from .service.book import Book, BookRepository, BookTable +from .service.product import Product, ProductRepository, ProductTable __all__ = [ "User", @@ -23,4 +25,10 @@ "UserIdentity", "UserIdentityTable", "UserIdentityRepository", + "Book", + "BookTable", + "BookRepository", + "Product", + "ProductTable", + "ProductRepository", ] diff --git a/src/app/entities/loader.py b/src/app/entities/loader.py new file mode 100644 index 0000000..5d1e7c5 --- /dev/null +++ b/src/app/entities/loader.py @@ -0,0 +1,72 @@ +"""Dynamic SQLModel table loader for Alembic migrations. + +This module provides automatic discovery and import of all SQLModel table +models, eliminating the need to manually update imports when new entities +are added. + +SQLModel uses SQLModel.metadata (equivalent to SQLAlchemy's declarative_base().metadata) +to track all registered tables. Models register themselves when imported. +This loader ensures all table.py modules are imported. +""" + +import importlib +from pathlib import Path + +from sqlalchemy import MetaData +from sqlmodel import SQLModel + + +def get_entities_path() -> Path: + """Get the path to the entities directory. + + Uses __file__ to be independent of package naming, + so it works after Copier renames 'src' to the project slug. + """ + return Path(__file__).parent + + +def load_all_tables() -> None: + """Dynamically import all table.py modules to register SQLModel tables. + + This function discovers all table.py files in the entities directory + and imports them. When a SQLModel class with `table=True` is imported, + it automatically registers with SQLModel.metadata. + + This approach eliminates the need to manually update imports when + adding new entities - just create the table.py file and it will + be discovered automatically. + """ + entities_path = get_entities_path() + + # Find this module's package name dynamically + # This handles Copier renaming 'src' to project slug + # __name__ will be something like 'src.app.entities.loader' or 'myproject.app.entities.loader' + package_base = __name__.rsplit(".", 2)[0] # e.g., 'src.app' or 'myproject.app' + + for table_file in entities_path.rglob("table.py"): + # Convert file path to module name relative to entities directory + relative_path = table_file.relative_to(entities_path) + # Remove .py extension and convert path separators to dots + module_parts = relative_path.with_suffix("").parts + module_name = f"{package_base}.entities.{'.'.join(module_parts)}" + + try: + importlib.import_module(module_name) + print(f"Imported tables from {module_name}") + except ImportError as e: + # Re-raise with more context for debugging + raise ImportError( + f"Failed to import table module '{module_name}' from {table_file}: {e}" + ) from e + + +def get_metadata() -> MetaData: + """Load all tables and return SQLModel.metadata for Alembic. + + This is the main entry point for Alembic's env.py. + + Returns: + SQLModel.metadata with all tables registered. + """ + load_all_tables() + return SQLModel.metadata diff --git a/src/app/runtime/config/config_data.py b/src/app/runtime/config/config_data.py index 71c3ea4..93ae3e1 100644 --- a/src/app/runtime/config/config_data.py +++ b/src/app/runtime/config/config_data.py @@ -6,10 +6,12 @@ from __future__ import annotations +from functools import cached_property from typing import Any, Literal from loguru import logger -from pydantic import BaseModel, Field, computed_field +from pydantic import BaseModel, Field, computed_field, model_validator +from sqlalchemy import URL def deep_freeze(value: Any) -> Any: @@ -153,6 +155,10 @@ class TemporalConfig(BaseModel): enabled: bool = Field(default=True, description="Enable temporal service") url: str = Field(default="temporal:7233", description="Temporal server url") namespace: str = Field(default="default", description="Temporal namespace") + db_user: str = Field(default="temporaluser", description="Temporal database user") + db_owner: str = Field( + default="temporalowner", description="Temporal database owner" + ) task_queue: str = Field(default="default", description="Temporal task queue name") worker: TemporalWorkerConfig = Field( default_factory=TemporalWorkerConfig, description="Worker configuration" @@ -173,6 +179,14 @@ class RedisConfig(BaseModel): password: str | None = Field( default=None, description="Password for Redis authentication" ) + password_file_path: str | None = Field( + default=None, + description="Path to file containing Redis password", + ) + password_env_var: str | None = Field( + default=None, + description="Environment variable name containing Redis password", + ) decode_responses: bool = Field( default=True, description="Decode Redis responses to strings" ) @@ -186,11 +200,35 @@ class RedisConfig(BaseModel): default=5, description="Socket connect timeout in seconds for Redis" ) + def _resolve_password(self) -> str | None: + """Resolve password from direct value, file, or environment variable.""" + import os + from pathlib import Path + + # Priority 1: Direct password value + if self.password: + return self.password + + # Priority 2: Password from file + if self.password_file_path: + file_path = Path(self.password_file_path) + if file_path.exists(): + return file_path.read_text(encoding="utf-8").strip() + + # Priority 3: Password from environment variable + if self.password_env_var: + env_password = os.environ.get(self.password_env_var) + if env_password: + return env_password + + return None + @computed_field # type: ignore[prop-decorator] @property def connection_string(self) -> str: """Construct the Redis connection string with password if provided.""" - if self.password: + resolved_password = self._resolve_password() + if resolved_password: if "@" in self.url: # URL already has auth info return self.url @@ -200,7 +238,7 @@ def connection_string(self) -> str: scheme, rest = parts # URL-encode password to handle special characters like /, @, :, etc. - encoded_password = quote_plus(self.password) + encoded_password = quote_plus(resolved_password) return f"{scheme}://:{encoded_password}@{rest}" return self.url @@ -356,6 +394,22 @@ class LoggingConfig(BaseModel): ) +class BundledPostgresConfig(BaseModel): + """Configuration for bundled PostgreSQL setup.""" + + enabled: bool = Field( + default=False, description="Enable bundled PostgreSQL configuration" + ) + password_file_path: str | None = Field( + default=None, + description="Path to file containing database password", + ) + password_env_var: str | None = Field( + default=None, + description="Environment variable name containing database password", + ) + + class DatabaseConfig(BaseModel): """Database configuration model.""" @@ -363,11 +417,21 @@ class DatabaseConfig(BaseModel): default="postgresql+asyncpg://user:password@postgres:5432/app_db", description="Database connection URL", ) + pg_superuser: str = Field( + default="postgres", description="Database superuser username" + ) + pg_db: str = Field(default="postgres", description="PostgreSQL database name") owner_user: str = Field(default="appowner", description="Database owner username") user: str = Field(default="user", description="Database username") ro_user: str = Field( default="backupuser", description="Database read-only username" ) + temporal_user: str = Field( + default="temporaluser", description="Temporal database user" + ) + temporal_owner: str = Field( + default="temporalowner", description="Temporal owner role" + ) app_db: str = Field(default="app_db", description="Database name") pool_size: int = Field(default=20, description="Connection pool size") environment_mode: str = Field( @@ -376,109 +440,132 @@ class DatabaseConfig(BaseModel): max_overflow: int = Field(default=10, description="Maximum pool overflow") pool_timeout: int = Field(default=30, description="Pool timeout in seconds") pool_recycle: int = Field(default=1800, description="Pool recycle time in seconds") - password_env_var: str | None = Field( - default=None, - description="Environment variable name containing database password", + bundled_postgres: BundledPostgresConfig = Field( + default_factory=BundledPostgresConfig, + description="Bundled PostgreSQL configuration", ) - password_file_path: str | None = Field( - default=None, - description="Path to file containing database password", - ) + @model_validator(mode="after") + def _clear_cached_properties(self) -> DatabaseConfig: + """Clear cached properties when the model is validated/modified. + + This ensures cached properties like parsed_url are recomputed + when the url field changes. + """ + # Clear the cached_property if it exists using the public API + try: + delattr(self, "parsed_url") + except AttributeError: + pass # Not cached yet, nothing to clear + return self + + @cached_property + def parsed_url(self) -> URL: + """Parse the database URL into its components.""" + from sqlalchemy.engine import make_url + + url_obj = make_url(self.url) + return url_obj + + @property + def host(self) -> str | None: + """Extract host from the database URL.""" + return self.parsed_url.host + + @property + def port(self) -> int | None: + """Extract port from the database URL.""" + return self.parsed_url.port @computed_field # type: ignore[prop-decorator] @property def password(self) -> str | None: """ - Get the database password from the appropriate source. - 1. If in development mode, try to parse from URL - 2. If in production mode, read from mounted secrets file specified - by `password_file_path` or environment variable specified by `password_env_var` + Resolve the database password. + + Uses the password from the URL if present, otherwise attempts to read + from environment variable or file as specified in bundled_postgres config. + + :return: The resolved database password, or None if not found. """ + + url_obj = self.parsed_url + if url_obj.password: + logger.debug("Using database password from URL") + return url_obj.password + password = None - if self.environment_mode == "development" or self.environment_mode == "test": - # In development mode, try to parse password from URL if present - from sqlalchemy.engine import make_url + if self.bundled_postgres.password_env_var: + logger.debug( + "Attempting to read database password from environment variable {}", + self.bundled_postgres.password_env_var, + ) - url_obj = make_url(self.url) - if url_obj.password: - password = url_obj.password + import os - elif self.environment_mode == "production": - # In production mode, read from file or environment variable + password = os.getenv(self.bundled_postgres.password_env_var) + if not password: + logger.debug( + "Environment variable {} for database password is not set or empty", + self.bundled_postgres.password_env_var, + ) + if not password and self.bundled_postgres.password_file_path: logger.debug( - "Attempting to read database password for production mode from environment variable {}", - self.password_env_var, + "Attempting to read database password from file {}", + self.bundled_postgres.password_file_path, ) - if self.password_env_var: - import os - - password = os.getenv(self.password_env_var) - if password: - return password - - if not password and self.password_file_path: - try: - with open(self.password_file_path, encoding="utf-8") as f: - password = f.read().strip() - except Exception as e: - logger.error( - "Failed to read database password from file {}: {}", - self.password_file_path, - e, - ) - raise - if not password: + try: + with open( + self.bundled_postgres.password_file_path, encoding="utf-8" + ) as f: + password = f.read().strip() + except Exception as e: logger.error( - "Database password not found in environment variable {} or file {}", - self.password_env_var, - self.password_file_path, + "Failed to read database password from file {}: {}", + self.bundled_postgres.password_file_path, + e, ) - raise ValueError("Database password not provided in production mode") - else: - raise ValueError( - "Invalid environment_mode; must be 'development', 'production', or 'test'" + if not password: + logger.error( + "Database password not found in environment variable {} or file {}", + self.bundled_postgres.password_env_var, + self.bundled_postgres.password_file_path, ) + + # Only raise error in production mode; dev/test can use passwordless SQLite + if self.environment_mode == "production": + raise ValueError("Database password not provided in production mode") + return password @computed_field # type: ignore[prop-decorator] @property def connection_string(self) -> str: """Construct the database connection string with password if provided.""" - from sqlalchemy.engine import make_url - - base_url = make_url(self.url) + base_url = self.parsed_url - # If the URL already has a password (development mode), use it as-is - if base_url.password: - # If in production mode, emit a warning if password is hardcoded - if self.environment_mode == "production": - logger.warning( - "Database URL contains a password in production mode; " - "consider using a secrets file or environment variable." - ) - - if self.password != base_url.password: - logger.warning( - "Database password from environment variable does not match the one in the URL. Using password from environment variable." - ) - base_url = base_url.set(password=self.password) + if self.password and base_url.password != self.password: + logger.warning( + "Database URL already contains a password that differs from the resolved password. Using password from environment variable or config." + ) + base_url = base_url.set(password=self.password) - if self.app_db != base_url.database: - logger.warning( - f"Database name '{self.app_db}' does not match the one in the URL '{base_url.database}'. Using '{self.app_db}'." - ) - base_url = base_url.set(database=self.app_db) + if self.app_db and base_url.database != self.app_db: + logger.warning( + f"Database name '{base_url.database}' in URL differs from configured app_db '{self.app_db}'. Using '{self.app_db}'." + ) + base_url = base_url.set(database=self.app_db) - if self.user != base_url.username: - logger.warning( - f"Database user '{self.user}' does not match the one in the URL '{base_url.username}'. Using '{self.user}'." - ) - base_url = base_url.set(username=self.user) + if self.user and base_url.username != self.user: + logger.warning( + f"Database user '{base_url.username}' in URL differs from configured user '{self.user}'. Using '{self.user}'." + ) + base_url = base_url.set(username=self.user) - # URL-encode password to handle special characters + # URL-encode password to handle special characters + if base_url.password: from urllib.parse import quote_plus password_to_encode = base_url.password or "" @@ -486,53 +573,28 @@ def connection_string(self) -> str: # Build base connection string conn_str = f"postgresql://{base_url.username}:{encoded_password}@{base_url.host}:{base_url.port}/{base_url.database}" - - # Add search_path=app ONLY for PostgreSQL in production mode - if self.environment_mode == "production" and base_url.drivername.startswith( - "postgresql" - ): - conn_str += "?options=-csearch_path%3Dapp" - - return conn_str - - # Otherwise, use the resolved password from the computed field and - # the resolved user and database from the URL or config - resolved_password = self.password - # If the user or database is not set in the config, fall back to the URL - resolved_user = self.user or base_url.username - resolved_db = self.app_db or base_url.database - - if resolved_user != base_url.username and self.user: - base_url = base_url.set(username=resolved_user) - if resolved_db != base_url.database and self.app_db: - base_url = base_url.set(database=resolved_db) - - if resolved_password: - from urllib.parse import quote_plus - - # URL-encode password to handle special characters - encoded_password = quote_plus(resolved_password) - # Build the connection string manually with encoded password - conn_str = f"postgresql://{resolved_user}:{encoded_password}@{base_url.host}:{base_url.port}/{resolved_db}" - - # Add search_path=app ONLY for PostgreSQL in production mode - if self.environment_mode == "production" and base_url.drivername.startswith( - "postgresql" - ): - conn_str += "?options=-csearch_path%3Dapp" - - return conn_str else: # Build connection string without password conn_str = f"postgresql://{base_url.username}@{base_url.host}:{base_url.port}/{base_url.database}" - # Add search_path=app ONLY for PostgreSQL in production mode - if self.environment_mode == "production" and base_url.drivername.startswith( - "postgresql" - ): - conn_str += "?options=-csearch_path%3Dapp" + # Preserve existing query parameters from original URL + if base_url.query: + from urllib.parse import urlencode + + # base_url.query is an immutable mapping, convert to dict + existing_params = dict(base_url.query) + conn_str += "?" + urlencode(existing_params) - return conn_str + # Add search_path=app ONLY for PostgreSQL in production mode + if self.environment_mode == "production" and base_url.drivername in ( + "postgresql", + "postgres", + ): + # Use & if query params already exist, otherwise ? + separator = "&" if base_url.query else "?" + conn_str += f"{separator}options=-csearch_path%3Dapp" + + return conn_str @computed_field # type: ignore[prop-decorator] @property diff --git a/src/app/runtime/config/config_loader.py b/src/app/runtime/config/config_loader.py index 0bedb36..feefdc7 100644 --- a/src/app/runtime/config/config_loader.py +++ b/src/app/runtime/config/config_loader.py @@ -145,11 +145,13 @@ def load_config( ) # Clear Redis password for development environment (dev Redis has no auth) - if env_mode == "development" and config.redis and config.redis.password: + if env_mode == "development" and config.redis: logger.info( "Clearing Redis password for development environment (dev Redis has no authentication)" ) config.redis.password = "" + config.redis.password_file_path = None + config.redis.password_env_var = None return config diff --git a/src/app/runtime/config/config_utils.py b/src/app/runtime/config/config_utils.py index d563f34..1d03e96 100644 --- a/src/app/runtime/config/config_utils.py +++ b/src/app/runtime/config/config_utils.py @@ -5,23 +5,21 @@ from loguru import logger -_SECRETS_LOADED = False - +from src.utils.paths import get_project_root -def _get_project_root() -> Path: - return Path(__file__).resolve().parents[4] +_SECRETS_LOADED = False def _candidate_secret_dirs() -> Iterable[Path]: custom_dir = os.getenv("SECRETS_KEYS_DIR") if custom_dir: yield Path(custom_dir) - project_root = _get_project_root() + project_root = get_project_root() yield project_root / "infra" / "secrets" / "keys" yield project_root / "secrets" / "keys" -def _load_secret_files_into_env() -> None: +def load_secret_files_into_env() -> None: global _SECRETS_LOADED if _SECRETS_LOADED: return @@ -43,7 +41,10 @@ def _load_secret_files_into_env() -> None: env_name = "".join(c if c.isalnum() or c == "_" else "_" for c in env_name) # Skip if empty name or already exists in environment - if not env_name or env_name in os.environ: + if not env_name: + continue + + if env_name in os.environ: continue # Check file size before reading @@ -88,6 +89,23 @@ def _load_secret_files_into_env() -> None: _SECRETS_LOADED = True +def _strip_inline_comment(value: str) -> str: + """ + Strip inline comments from environment variable values. + + Handles comments starting with # (ignoring escaped \\#). + Preserves the value before the comment and strips trailing whitespace. + + Examples: + "3600 # Session max age" -> "3600" + "value # comment" -> "value" + "no comment here" -> "no comment here" + """ + # Find first unescaped # character + comment_pattern = r"\s*(? str: """ Substitute environment variable placeholders in text. @@ -96,9 +114,11 @@ def substitute_env_vars(text: str) -> str: - ${VAR_NAME} - required variable (raises error if missing) - ${VAR_NAME:-default} - optional with default value - ${VAR_NAME:?error_message} - required with custom error message + + Note: Environment variable values are automatically stripped of inline comments. """ - _load_secret_files_into_env() + load_secret_files_into_env() def replacer(match: re.Match[str]) -> str: var_expr = match.group(1) @@ -106,7 +126,8 @@ def replacer(match: re.Match[str]) -> str: # Handle default values: ${VAR:-default} if ":-" in var_expr: var_name, default = var_expr.split(":-", 1) - return os.getenv(var_name, default) + value = os.getenv(var_name, default) + return _strip_inline_comment(value) # Handle error messages: ${VAR:?message} elif ":?" in var_expr: @@ -116,7 +137,7 @@ def replacer(match: re.Match[str]) -> str: raise ValueError( f"Required environment variable {var_name}: {error_msg}" ) - return value + return _strip_inline_comment(value) # Handle required variables: ${VAR} else: @@ -124,7 +145,7 @@ def replacer(match: re.Match[str]) -> str: value = os.getenv(var_name) if value is None: raise ValueError(f"Required environment variable {var_name} not set") - return value + return _strip_inline_comment(value) # Match ${...} patterns pattern = r"\$\{([^}]+)\}" diff --git a/src/cli/__init__.py b/src/cli/__init__.py index b1cd3d1..eeabfe4 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -6,8 +6,8 @@ Command Groups: - dev: Development Docker Compose environment -- prod: Production Docker Compose deployment -- k8s: Kubernetes Helm deployment +- prod: Production Docker Compose deployment (includes 'prod db' subcommands) +- k8s: Kubernetes Helm deployment (includes 'k8s db' subcommands) - fly: Fly.io Kubernetes (coming soon) - entity: Entity/model scaffolding - secrets: Secret management @@ -25,6 +25,7 @@ secrets_app, users_app, ) +from .context import build_cli_context # Create the main CLI application app = typer.Typer( @@ -33,6 +34,14 @@ rich_markup_mode="rich", ) + +@app.callback() +def _configure_context(ctx: typer.Context) -> None: + """Attach runtime dependencies to the CLI context.""" + if ctx.obj is None: + ctx.obj = build_cli_context() + + # Register deployment target command groups app.add_typer(dev_app, name="dev", help="Development environment commands") app.add_typer(prod_app, name="prod", help="Production Docker Compose commands") diff --git a/src/cli/commands/__init__.py b/src/cli/commands/__init__.py index 8d79da5..671bdf9 100644 --- a/src/cli/commands/__init__.py +++ b/src/cli/commands/__init__.py @@ -5,8 +5,8 @@ Command Groups: - dev: Development environment using Docker Compose -- prod: Production Docker Compose deployment -- k8s: Kubernetes deployment using Helm +- prod: Production Docker Compose deployment (includes 'prod db' subcommands) +- k8s: Kubernetes deployment using Helm (includes 'k8s db' subcommands) - fly: Fly.io Kubernetes (FKS) deployment (future) - entity: Entity/model scaffolding - secrets: Secret management utilities diff --git a/src/cli/commands/db/__init__.py b/src/cli/commands/db/__init__.py new file mode 100644 index 0000000..b253600 --- /dev/null +++ b/src/cli/commands/db/__init__.py @@ -0,0 +1,28 @@ +"""Database CLI workflow helpers.""" + +from .runtime import DbRuntime, no_port_forward +from .runtime_compose import get_compose_runtime +from .runtime_k8s import get_k8s_runtime +from .workflows import ( + run_backup, + run_init, + run_migrate, + run_reset, + run_status, + run_sync, + run_verify, +) + +__all__ = [ + "DbRuntime", + "no_port_forward", + "get_compose_runtime", + "get_k8s_runtime", + "run_backup", + "run_init", + "run_migrate", + "run_reset", + "run_status", + "run_sync", + "run_verify", +] diff --git a/src/cli/commands/db/runtime.py b/src/cli/commands/db/runtime.py new file mode 100644 index 0000000..a173e6d --- /dev/null +++ b/src/cli/commands/db/runtime.py @@ -0,0 +1,32 @@ +"""Database runtime adapters for CLI workflows.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from src.cli.shared.console import CLIConsole +from src.infra.postgres import PostgresConnection + + +@dataclass(frozen=True) +class DbRuntime: + """Environment-specific behaviors for database workflows.""" + + name: str + console: CLIConsole + get_settings: Callable[[], Any] + connect: Callable[[Any, bool], PostgresConnection] + port_forward: Callable[[], AbstractContextManager[None]] + get_deployer: Callable[[], Any] + secrets_dirs: Sequence[Path] + is_temporal_enabled: Callable[[], bool] + is_bundled_postgres_enabled: Callable[[], bool] + + +def no_port_forward() -> AbstractContextManager[None]: + """No-op context manager for environments without port forwarding.""" + return nullcontext() diff --git a/src/cli/commands/db/runtime_compose.py b/src/cli/commands/db/runtime_compose.py new file mode 100644 index 0000000..5a7e845 --- /dev/null +++ b/src/cli/commands/db/runtime_compose.py @@ -0,0 +1,32 @@ +"""Docker Compose runtime for database workflows.""" + +from __future__ import annotations + +from src.cli.commands.db.runtime import DbRuntime, no_port_forward +from src.cli.deployment.prod_deployer import get_deployer +from src.cli.deployment.status_display import is_temporal_enabled +from src.cli.shared.console import console +from src.infra.docker_compose.postgres_connection import ( + get_docker_compose_postgres_connection, +) +from src.infra.postgres.connection import get_settings +from src.infra.utils.service_config import is_bundled_postgres_enabled +from src.utils.paths import get_project_root + + +def get_compose_runtime() -> DbRuntime: + """Build a DbRuntime for Docker Compose (prod) workflows.""" + project_root = get_project_root() + return DbRuntime( + name="compose", + console=console, + get_settings=get_settings, + connect=lambda settings, superuser: get_docker_compose_postgres_connection( + settings, superuser_mode=superuser + ), + port_forward=no_port_forward, + get_deployer=get_deployer, + secrets_dirs=[project_root / "infra" / "secrets" / "keys"], + is_temporal_enabled=is_temporal_enabled, + is_bundled_postgres_enabled=is_bundled_postgres_enabled, + ) diff --git a/src/cli/commands/db/runtime_k8s.py b/src/cli/commands/db/runtime_k8s.py new file mode 100644 index 0000000..226f892 --- /dev/null +++ b/src/cli/commands/db/runtime_k8s.py @@ -0,0 +1,40 @@ +"""Kubernetes runtime for database workflows.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager + +from src.cli.commands.db.runtime import DbRuntime +from src.cli.deployment.helm_deployer.deployer import get_deployer +from src.cli.deployment.status_display import is_temporal_enabled +from src.cli.shared.console import console +from src.infra.k8s import get_namespace, get_postgres_label +from src.infra.k8s.port_forward import postgres_port_forward_if_needed +from src.infra.k8s.postgres_connection import get_k8s_postgres_connection +from src.infra.postgres.connection import get_settings +from src.infra.utils.service_config import is_bundled_postgres_enabled +from src.utils.paths import get_project_root + + +def _port_forward() -> AbstractContextManager[None]: + namespace = get_namespace() + label = get_postgres_label() + return postgres_port_forward_if_needed(namespace=namespace, pod_label=label) + + +def get_k8s_runtime() -> DbRuntime: + """Build a DbRuntime for Kubernetes workflows.""" + project_root = get_project_root() + return DbRuntime( + name="k8s", + console=console, + get_settings=get_settings, + connect=lambda settings, superuser: get_k8s_postgres_connection( + settings, superuser_mode=superuser + ), + port_forward=_port_forward, + get_deployer=get_deployer, + secrets_dirs=[project_root / "infra" / "secrets" / "keys"], + is_temporal_enabled=is_temporal_enabled, + is_bundled_postgres_enabled=is_bundled_postgres_enabled, + ) diff --git a/src/cli/commands/db/workflows.py b/src/cli/commands/db/workflows.py new file mode 100644 index 0000000..835d2a5 --- /dev/null +++ b/src/cli/commands/db/workflows.py @@ -0,0 +1,249 @@ +"""Shared database workflows for CLI commands.""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from src.cli.commands.db.runtime import DbRuntime +from src.infra.postgres.migrations import run_migration + + +def run_init(runtime: DbRuntime) -> bool: + from src.infra.postgres import PostgresInitializer + + settings = runtime.get_settings().ensure_all_passwords() + with runtime.port_forward(): + with runtime.connect(settings, True) as conn: + initializer = PostgresInitializer(connection=conn) + return initializer.initialize() + + +def run_verify(runtime: DbRuntime, *, superuser_mode: bool) -> bool: + from src.infra.postgres import PostgresVerifier + + settings = runtime.get_settings().ensure_all_passwords() + with runtime.port_forward(): + with runtime.connect(settings, superuser_mode) as conn: + verifier = PostgresVerifier(connection=conn) + return verifier.verify_all() + + +def run_sync(runtime: DbRuntime) -> bool: + from src.infra.postgres import PostgresPasswordSync + + settings = runtime.get_settings() + success = True + + if runtime.is_bundled_postgres_enabled(): + runtime.console.print_subheader("Syncing bundled PostgreSQL superuser password") + with runtime.port_forward(): + with runtime.connect(settings, True) as conn: + sync_tool = PostgresPasswordSync( + connection=conn, + deployer=runtime.get_deployer(), + secrets_dirs=list(runtime.secrets_dirs), + ) + success = sync_tool.sync_bundled_superuser_password() + + with runtime.port_forward(): + with runtime.connect(settings, True) as conn: + sync_tool = PostgresPasswordSync( + connection=conn, + deployer=runtime.get_deployer(), + secrets_dirs=list(runtime.secrets_dirs), + ) + runtime.console.print_subheader( + "Syncing application database user roles and passwords" + ) + user_success = sync_tool.sync_user_roles_and_passwords() + success = success and user_success + + return success + + +def run_backup( + runtime: DbRuntime, *, output_dir: Path, superuser_mode: bool +) -> tuple[bool, str]: + from src.infra.postgres import PostgresBackup + + settings = runtime.get_settings() + with runtime.port_forward(): + with runtime.connect(settings, superuser_mode) as conn: + backup_tool = PostgresBackup( + connection=conn, + backup_dir=output_dir, + ) + return backup_tool.create_backup() + + +def run_reset( + runtime: DbRuntime, *, include_temporal: bool, superuser_mode: bool +) -> bool: + from src.infra.postgres import PostgresReset + + settings = runtime.get_settings() + with runtime.port_forward(): + with runtime.connect(settings, superuser_mode) as conn: + reset_tool = PostgresReset(connection=conn) + return reset_tool.reset(include_temporal=include_temporal) + + +def run_status(runtime: DbRuntime, *, superuser_mode: bool) -> None: + import time + + from rich.table import Table + + settings = runtime.get_settings() + + try: + start = time.perf_counter() + with runtime.port_forward(): + with runtime.connect(settings, superuser_mode) as conn: + latency_ms = (time.perf_counter() - start) * 1000 + + perf_table = Table(title="Connection & Performance") + perf_table.add_column("Metric", style="cyan") + perf_table.add_column("Value", style="green") + + perf_table.add_row("Host", f"{settings.host}:{settings.port}") + perf_table.add_row("Connection Latency", f"{latency_ms:.2f} ms") + + try: + uptime = conn.scalar( + "SELECT EXTRACT(EPOCH FROM (now() - pg_postmaster_start_time()));" + ) + if uptime: + hours = int(uptime // 3600) + minutes = int((uptime % 3600) // 60) + perf_table.add_row("Uptime", f"{hours}h {minutes}m") + except Exception: + pass + + try: + active_conns = conn.scalar( + "SELECT count(*) FROM pg_stat_activity WHERE state = 'active';" + ) + total_conns = conn.scalar("SELECT count(*) FROM pg_stat_activity;") + max_conns = conn.scalar( + "SELECT setting::int FROM pg_settings WHERE name = 'max_connections';" + ) + perf_table.add_row( + "Connections", + f"{active_conns} active / {total_conns} total / {max_conns} max", + ) + except Exception: + pass + + try: + cache_hit = conn.scalar( + """ + SELECT ROUND( + 100.0 * sum(blks_hit) / NULLIF(sum(blks_hit) + sum(blks_read), 0), + 2 + ) + FROM pg_stat_database; + """ + ) + if cache_hit is not None: + color = ( + "green" + if cache_hit >= 90 + else "yellow" + if cache_hit >= 75 + else "red" + ) + perf_table.add_row( + "Cache Hit Ratio", f"[{color}]{cache_hit}%[/{color}]" + ) + except Exception: + pass + + runtime.console.print(perf_table) + + size_table = Table(title="\nDatabase Sizes") + size_table.add_column("Database", style="cyan") + size_table.add_column("Size", style="green") + size_table.add_column("Tables", style="blue") + + databases_to_check = [settings.app_db] + if runtime.is_temporal_enabled(): + databases_to_check.extend(["temporal", "temporal_visibility"]) + + for db_name in databases_to_check: + try: + db_size = conn.scalar( + "SELECT pg_size_pretty(pg_database_size(%s));", + (db_name,), + ) + table_count = conn.scalar( + """ + SELECT count(*) + FROM pg_catalog.pg_tables + WHERE schemaname NOT IN ('pg_catalog', 'information_schema'); + """, + database=db_name, + ) + row_count_query = """ + SELECT SUM(n_live_tup) + FROM pg_stat_user_tables; + """ + row_count = conn.scalar(row_count_query, database=db_name) or 0 + + if table_count and table_count > 0: + size_table.add_row( + db_name, + db_size or "unknown", + f"{table_count} tables, ~{row_count:,} rows", + ) + else: + size_table.add_row( + db_name, + db_size or "unknown", + "[dim]no tables[/dim]", + ) + except Exception: + pass + + runtime.console.print(size_table) + + except Exception as exc: + runtime.console.error(f"Failed to connect: {exc}") + runtime.console.print( + "\n[dim]Check your database configuration and connectivity[/dim]" + ) + runtime.console.print(f"[dim]Host: {settings.host}:{settings.port}[/dim]") + + +def run_migrate( + runtime: DbRuntime, + *, + action: str, + revision: str | None, + message: str | None, + merge_revisions: list[str], + purge: bool, + autogenerate: bool, + sql: bool, +) -> None: + settings = runtime.get_settings().ensure_superuser_password() + + with runtime.port_forward(): + conn = runtime.connect(settings, True) + database_url = conn.get_connection_string() + + success = run_migration( + action=action, + revision=revision, + message=message, + merge_revisions=merge_revisions, + purge=purge, + autogenerate=autogenerate, + sql=sql, + database_url=database_url, + console=runtime.console, + ) + + if not success: + raise typer.Exit(1) diff --git a/src/cli/commands/db_utils.py b/src/cli/commands/db_utils.py new file mode 100644 index 0000000..f38c2db --- /dev/null +++ b/src/cli/commands/db_utils.py @@ -0,0 +1,417 @@ +"""Shared database management utilities for both prod and k8s deployments. + +This module contains common functionality used by both Docker Compose (prod) +and Kubernetes (k8s) database management commands. +""" + +import typer + +from src.cli.shared.console import console +from src.utils.paths import get_project_root + + +def parse_connection_string(conn_str: str) -> dict[str, str | None]: + """Parse a PostgreSQL connection string into components. + + Supports formats: + - postgres://user:pass@host:port/database?params + - postgresql://user:pass@host:port/database?params + + Args: + conn_str: PostgreSQL connection string + + Returns: + Dictionary with keys: username, password, host, port, database, sslmode + """ + from urllib.parse import parse_qs, urlparse + + parsed = urlparse(conn_str) + + result: dict[str, str | None] = { + "username": parsed.username, + "password": parsed.password, + "host": parsed.hostname, + "port": str(parsed.port) if parsed.port else None, + "database": parsed.path.lstrip("/") if parsed.path else None, + "sslmode": None, + } + + # Parse query params for sslmode + if parsed.query: + params = parse_qs(parsed.query) + if "sslmode" in params: + result["sslmode"] = params["sslmode"][0] + + return result + + +def build_connection_string( + *, + username: str, + host: str, + port: str, + database: str, + sslmode: str | None = None, + include_password: bool = False, + password: str | None = None, +) -> str: + """Build a PostgreSQL connection string from components. + + Args: + username: Database username + host: Database host + port: Database port + database: Database name + sslmode: SSL mode (e.g., 'require', 'verify-full') + include_password: Whether to include password in connection string + password: Database password (only used if include_password=True) + + Returns: + PostgreSQL connection string + """ + url = f"postgres://{username}" + if include_password and password: + url += f":{password}" + url += f"@{host}:{port}/{database}" + if sslmode: + url += f"?sslmode={sslmode}" + return url + + +def update_env_file(updates: dict[str, str]) -> None: + """Update specific keys in the .env file. + + Updates existing keys or appends new ones if not found. + Preserves comments and formatting of other lines. + + Args: + updates: Dictionary of key-value pairs to update + + Raises: + typer.Exit: If .env file is not found + """ + env_path = get_project_root() / ".env" + + if not env_path.exists(): + console.print("[red]❌ .env file not found[/red]") + raise typer.Exit(1) + + lines = env_path.read_text().splitlines() + updated_keys: set[str] = set() + + # Update existing keys + new_lines = [] + for line in lines: + stripped = line.strip() + if stripped and not stripped.startswith("#") and "=" in stripped: + key = stripped.split("=", 1)[0] + if key in updates: + new_lines.append(f"{key}={updates[key]}") + updated_keys.add(key) + continue + new_lines.append(line) + + # Add any keys that weren't found + for key, value in updates.items(): + if key not in updated_keys: + new_lines.append(f"{key}={value}") + + env_path.write_text("\n".join(new_lines) + "\n") + + +def read_env_example_values(keys: list[str]) -> dict[str, str]: + """Read specific values from .env.example file. + + Args: + keys: List of environment variable keys to read + + Returns: + Dictionary of key-value pairs found in .env.example + + Raises: + typer.Exit: If .env.example file is not found + """ + env_example_path = get_project_root() / ".env.example" + + if not env_example_path.exists(): + console.print("[red]❌ .env.example file not found[/red]") + raise typer.Exit(1) + + values: dict[str, str] = {} + for line in env_example_path.read_text().splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#") and "=" in stripped: + key, value = stripped.split("=", 1) + if key in keys: + values[key] = value + + return values + + +def save_password_to_secrets(password: str) -> None: + """Save password to secrets file, backing up existing file if present. + + Args: + password: Password to save + + Creates: + - infra/secrets/keys/postgres_password.txt with mode 0600 + - infra/secrets/keys/postgres_password.txt.bak (backup if file exists) + """ + secrets_dir = get_project_root() / "infra" / "secrets" / "keys" + secrets_dir.mkdir(parents=True, exist_ok=True) + + password_file = secrets_dir / "postgres_password.txt" + + # Backup existing file + if password_file.exists(): + backup_file = secrets_dir / "postgres_password.txt.bak" + import shutil + + shutil.copy2(password_file, backup_file) + console.print(f"[dim]Backed up existing password to {backup_file.name}[/dim]") + + password_file.write_text(password) + # Set restrictive permissions + password_file.chmod(0o600) + + +def update_bundled_postgres_config(enabled: bool) -> None: + """Update bundled_postgres.enabled in config.yaml. + + Args: + enabled: Whether bundled PostgreSQL is enabled + + Modifies: + config.yaml - sets database.bundled_postgres.enabled + """ + from src.app.runtime.config.config_loader import load_config, save_config + + config = load_config(processed=False) + if isinstance(config, dict): + config["config"]["database"]["bundled_postgres"]["enabled"] = enabled + save_config(config) + else: + # It's a ConfigData object + config.database.bundled_postgres.enabled = enabled + save_config(config) + + +def validate_external_db_params( + *, + connection_string: str | None, + host: str | None, + port: str | None, + username: str | None, + password: str | None, + database: str | None, + sslmode: str | None, +) -> tuple[str, str, str, str, str, str | None]: + """Validate and merge external database connection parameters. + + Parses connection string if provided, then applies parameter overrides. + Validates that all required parameters are present. + + Args: + connection_string: Optional connection string to parse + host: Optional host override + port: Optional port override + username: Optional username override + password: Optional password override + database: Optional database override + sslmode: Optional SSL mode override + + Returns: + Tuple of (host, port, username, password, database, sslmode) + + Raises: + typer.Exit: If required parameters are missing + """ + # Parse connection string if provided + parsed: dict[str, str | None] = {} + if connection_string: + console.print("[cyan]ℹ[/cyan] Parsing connection string...") + parsed = parse_connection_string(connection_string) + + # Apply overrides (standalone params take precedence) + final_host = host or parsed.get("host") + final_port = port or parsed.get("port") or "5432" + final_username = username or parsed.get("username") + final_password = password or parsed.get("password") + final_database = database or parsed.get("database") + final_sslmode = sslmode or parsed.get("sslmode") + + # Validate required fields + missing = [] + if not final_host: + missing.append("host") + if not final_username: + missing.append("username") + if not final_password: + missing.append("password") + if not final_database: + missing.append("database") + + if missing: + console.error(f"Missing required parameters: {', '.join(missing)}") + console.print( + "[dim]Provide via --connection-string or individual options[/dim]" + ) + raise typer.Exit(1) + + # Type narrowing for mypy + assert final_host and final_username and final_password and final_database + + return ( + final_host, + final_port, + final_username, + final_password, + final_database, + final_sslmode, + ) + + +def configure_external_database( + *, + connection_string: str | None, + host: str | None, + port: str | None, + username: str | None, + password: str | None, + database: str | None, + sslmode: str | None, + tls_ca: str | None, + next_steps_cmd_prefix: str, +) -> None: + """Configure connection to an external PostgreSQL instance. + + This is the complete workflow for setting up an external database: + 1. Parse and validate connection parameters + 2. Update .env file + 3. Save password to secrets + 4. Update config.yaml + 5. Copy TLS CA certificate if provided + + Args: + connection_string: Optional PostgreSQL connection string + host: Optional database host + port: Optional database port + username: Optional database username + password: Optional database password + database: Optional database name + sslmode: Optional SSL mode + tls_ca: Optional path to TLS CA certificate file + next_steps_cmd_prefix: Command prefix for next steps (e.g., "prod" or "k8s") + """ + import shutil + from pathlib import Path + + console.print_header("Configuring External PostgreSQL") + + # Validate and merge parameters + ( + final_host, + final_port, + final_username, + final_password, + final_database, + final_sslmode, + ) = validate_external_db_params( + connection_string=connection_string, + host=host, + port=port, + username=username, + password=password, + database=database, + sslmode=sslmode, + ) + + # Build connection string WITHOUT password + conn_url = build_connection_string( + username=final_username, + host=final_host, + port=final_port, + database=final_database, + sslmode=final_sslmode, + include_password=False, + ) + + console.print(f"[cyan]ℹ[/cyan] Host: {final_host}:{final_port}") + console.print(f"[cyan]ℹ[/cyan] User: {final_username}") + console.print(f"[cyan]ℹ[/cyan] Database: {final_database}") + if final_sslmode: + console.print(f"[cyan]ℹ[/cyan] SSL Mode: {final_sslmode}") + + # Step 1: Update .env file + console.print("\n[cyan]ℹ[/cyan] Updating .env file...") + update_env_file( + { + "PRODUCTION_DATABASE_URL": conn_url, + "PG_SUPERUSER": final_username, + "PG_DB": final_database, + } + ) + console.ok(".env file updated") + + # Step 2: Save password to secrets file + console.print("[cyan]ℹ[/cyan] Saving password to secrets...") + save_password_to_secrets(final_password) + console.ok("Password saved to infra/secrets/keys/postgres_password.txt") + + # Step 3: Update config.yaml + console.info("Updating config.yaml...") + update_bundled_postgres_config(enabled=False) + console.ok("config.yaml updated (bundled_postgres.enabled=false)") + + # Step 4: Copy TLS CA certificate if provided + if tls_ca: + console.print("[cyan]ℹ[/cyan] Setting up TLS CA certificate...") + tls_ca_path = Path(tls_ca) + if not tls_ca_path.exists(): + console.error(f"TLS CA file not found: {tls_ca}") + raise typer.Exit(1) + + certs_dir = get_project_root() / "infra" / "secrets" / "certs" + certs_dir.mkdir(parents=True, exist_ok=True) + + # Copy to dedicated external postgres CA file + external_ca_file = certs_dir / "ca-bundle-postgres-external.crt" + shutil.copy2(tls_ca_path, external_ca_file) + external_ca_file.chmod(0o644) + console.ok( + f"Copied TLS CA to {external_ca_file.relative_to(get_project_root())}" + ) + + # Append to ca-bundle.crt + ca_bundle_file = certs_dir / "ca-bundle.crt" + if ca_bundle_file.exists(): + # Read existing content + existing_content = ca_bundle_file.read_text() + external_ca_content = external_ca_file.read_text() + + # Check if external CA is already in bundle + if external_ca_content.strip() not in existing_content: + # Append external CA to bundle + with ca_bundle_file.open("a") as f: + f.write("\n# External PostgreSQL CA Certificate\n") + f.write(external_ca_content) + console.ok("Appended external CA to ca-bundle.crt") + else: + console.print("[dim]External CA already present in ca-bundle.crt[/dim]") + else: + # Create ca-bundle.crt with just the external CA + with ca_bundle_file.open("w") as f: + f.write("# External PostgreSQL CA Certificate\n") + f.write(external_ca_file.read_text()) + ca_bundle_file.chmod(0o644) + console.ok("Created ca-bundle.crt with external CA") + + console.print("\n[bold green]🎉 External PostgreSQL configured![/bold green]") + console.print("\n[dim]Next steps:[/dim]") + console.print( + f" 1. Run 'uv run api-forge-cli {next_steps_cmd_prefix} db init' to initialize database" + ) + console.print( + f" 2. Run 'uv run api-forge-cli {next_steps_cmd_prefix} db verify' to verify setup" + ) diff --git a/src/cli/commands/dev.py b/src/cli/commands/dev.py index aeb0a1d..127ddbd 100644 --- a/src/cli/commands/dev.py +++ b/src/cli/commands/dev.py @@ -11,20 +11,16 @@ restart - Restart a specific service """ +import subprocess from pathlib import Path import typer from src.cli.deployment import DevDeployer from src.cli.deployment.helm_deployer.image_builder import DeploymentError - -from .shared import ( - confirm_action, - console, - get_project_root, - handle_error, - print_header, -) +from src.cli.shared.compose import ComposeRunner +from src.cli.shared.console import console +from src.utils.paths import get_project_root # Create the dev command group app = typer.Typer( @@ -39,6 +35,15 @@ def _get_deployer() -> DevDeployer: return DevDeployer(console, Path(get_project_root())) +def _get_compose_runner() -> ComposeRunner: + """Create a ComposeRunner for the dev compose file.""" + project_root = Path(get_project_root()) + return ComposeRunner( + project_root, + compose_file=project_root / "docker-compose.dev.yml", + ) + + # ============================================================================= # Commands # ============================================================================= @@ -78,13 +83,13 @@ def up( # Force restart all services api-forge-cli dev up --force """ - print_header("Starting Development Environment") + console.print_header("Starting Development Environment") try: deployer = _get_deployer() deployer.deploy(force=force, no_wait=no_wait, start_server=start_server) except DeploymentError as e: - handle_error(f"Deployment failed: {e.message}", e.details) + console.handle_error(f"Deployment failed: {e.message}", e.details) @app.command() @@ -123,7 +128,7 @@ def down( " This includes databases, caches, and any persistent storage." ) - if not confirm_action( + if not console.confirm_action( action="Stop development environment", details=details, extra_warning=extra_warning, @@ -132,13 +137,13 @@ def down( console.print("[dim]Operation cancelled.[/dim]") raise typer.Exit(0) - print_header("Stopping Development Environment", style="red") + console.print_header("Stopping Development Environment", style="red") try: deployer = _get_deployer() deployer.teardown(volumes=volumes) except DeploymentError as e: - handle_error(f"Teardown failed: {e.message}", e.details) + console.handle_error(f"Teardown failed: {e.message}", e.details) @app.command() @@ -189,17 +194,6 @@ def logs( # Follow Keycloak logs api-forge-cli dev logs keycloak --follow """ - import subprocess - - compose_file = "docker-compose.dev.yml" - cmd = ["docker", "compose", "-f", compose_file, "logs"] - - if tail: - cmd.extend(["--tail", str(tail)]) - - if follow: - cmd.append("--follow") - if service: # Map friendly names to Docker Compose service names service_map = { @@ -210,12 +204,14 @@ def logs( "temporal-ui": "temporal-web", } compose_service = service_map.get(service.lower(), service) - cmd.append(compose_service) + else: + compose_service = None try: - subprocess.run(cmd, cwd=get_project_root(), check=True) + runner = _get_compose_runner() + runner.logs(service=compose_service, follow=follow, tail=tail) except subprocess.CalledProcessError as e: - handle_error(f"Failed to get logs: {e}") + console.handle_error(f"Failed to get logs: {e}") except KeyboardInterrupt: pass # User cancelled with Ctrl+C @@ -238,10 +234,6 @@ def restart( # Restart Keycloak api-forge-cli dev restart keycloak """ - import subprocess - - compose_file = "docker-compose.dev.yml" - # Map friendly names to Docker Compose service names service_map = { "keycloak": "keycloak", @@ -255,10 +247,9 @@ def restart( console.print(f"[bold]Restarting {service}...[/bold]") - cmd = ["docker", "compose", "-f", compose_file, "restart", compose_service] - try: - subprocess.run(cmd, cwd=get_project_root(), check=True) + runner = _get_compose_runner() + runner.restart(service=compose_service) console.print(f"[green]✅ {service} restarted successfully[/green]") except subprocess.CalledProcessError as e: - handle_error(f"Failed to restart {service}: {e}") + console.handle_error(f"Failed to restart {service}: {e}") diff --git a/src/cli/commands/entity/__init__.py b/src/cli/commands/entity/__init__.py new file mode 100644 index 0000000..fb7156e --- /dev/null +++ b/src/cli/commands/entity/__init__.py @@ -0,0 +1,5 @@ +"""Entity CLI package.""" + +from .cli import entity_app + +__all__ = ["entity_app"] diff --git a/src/cli/commands/entity.py b/src/cli/commands/entity/cli.py similarity index 55% rename from src/cli/commands/entity.py rename to src/cli/commands/entity/cli.py index c0441ef..4e60824 100644 --- a/src/cli/commands/entity.py +++ b/src/cli/commands/entity/cli.py @@ -1,179 +1,30 @@ """Entity management CLI commands.""" -import re +from __future__ import annotations + from pathlib import Path -from typing import Any import typer -from jinja2 import Environment, FileSystemLoader from rich.panel import Panel from rich.prompt import Prompt from rich.table import Table -from .shared import console, get_project_root +from src.cli.shared.console import console +from src.utils.paths import get_project_root + +from .scaffold import ( + create_crud_router, + create_entity_files, + prompt_for_fields, + register_router_with_app, + sanitize_entity_name, + unregister_router_from_app, +) # Create the entity command group entity_app = typer.Typer(help="🎭 Entity management commands") -def get_template_env() -> Environment: - """Get Jinja2 environment for template rendering.""" - template_dir = Path(__file__).parent / "templates" - return Environment(loader=FileSystemLoader(template_dir)) - - -def render_template_to_file( - template_name: str, output_path: Path, context: dict[str, Any] -) -> None: - """Render a Jinja2 template to a file.""" - env = get_template_env() - template = env.get_template(template_name) - content = template.render(**context) - output_path.write_text(content) - - -def sanitize_entity_name(name: str) -> str: - """Sanitize entity name to conform to Python naming conventions.""" - # Convert to PascalCase for class names - # Remove special characters and split on non-alphanumeric - words = re.findall(r"[a-zA-Z0-9]+", name) - return "".join(word.capitalize() for word in words) - - -def sanitize_field_name(name: str) -> str: - """Sanitize field name to conform to Python snake_case conventions.""" - # Convert to snake_case for field names - words = re.findall(r"[a-zA-Z0-9]+", name) - return "_".join(word.lower() for word in words) - - -def prompt_for_fields() -> list[dict[str, str | bool]]: - """Prompt user for entity fields.""" - fields: list[dict[str, str | bool]] = [] - console.print( - "\n[blue]Define entity fields (press Enter without a name to finish):[/blue]" - ) - - while True: - field_name = Prompt.ask("[cyan]Field name", default="") - if not field_name.strip(): - break - - field_name = sanitize_field_name(field_name) - - field_type = Prompt.ask( - f"[cyan]Type for '{field_name}'", - choices=["str", "int", "float", "bool", "datetime"], - default="str", - ) - - optional = ( - Prompt.ask( - f"[cyan]Is '{field_name}' optional?", choices=["y", "n"], default="n" - ) - == "y" - ) - - description = Prompt.ask( - f"[cyan]Description for '{field_name}'", - default=f"{field_name.replace('_', ' ').title()}", - ) - - fields.append( - { - "name": field_name, - "type": field_type, - "optional": optional, - "description": description, - } - ) - - console.print(f"[green]✓[/green] Added field: {field_name}: {field_type}") - - return fields - - -def create_entity_files( - entity_name: str, fields: list[dict[str, str | bool]], package_path: Path -) -> None: - """Create all entity files using Jinja2 templates.""" - context = {"entity_name": entity_name, "fields": fields} - - # Create all files from templates - render_template_to_file("entity.py.j2", package_path / "entity.py", context) - render_template_to_file("table.py.j2", package_path / "table.py", context) - render_template_to_file("repository.py.j2", package_path / "repository.py", context) - render_template_to_file("__init__.py.j2", package_path / "__init__.py", context) - - -def create_crud_router(entity_name: str, fields: list[dict[str, str | bool]]) -> None: - """Create a CRUD router for the entity using templates.""" - router_dir = ( - get_project_root() / "src" / "app" / "api" / "http" / "routers" / "service" - ) - router_dir.mkdir(exist_ok=True) - - # Create router file from template - router_file = router_dir / f"{entity_name.lower()}.py" - context = {"entity_name": entity_name, "fields": fields} - render_template_to_file("router.py.j2", router_file, context) - - # Update routers __init__.py if it exists - routers_init = router_dir / "__init__.py" - if not routers_init.exists(): - routers_init.write_text('"""Service routers package."""\n') - - -def register_router_with_app(entity_name: str) -> None: - """Add import and registration for the new router in app.py.""" - app_file = get_project_root() / "src" / "app" / "api" / "http" / "app.py" - - # Read current content - content = app_file.read_text() - - # Add import - import_line = f"from src.app.api.http.routers.service.{entity_name.lower()} import router as {entity_name.lower()}_router" - - # Find the last router import to add after it - lines = content.split("\n") - import_insert_idx = -1 - - for i, line in enumerate(lines): - if "from src.app.api.http.routers" in line and "import router" in line: - import_insert_idx = i + 1 - - if import_insert_idx > 0: - lines.insert(import_insert_idx, import_line) - else: - # Find other imports and add after them - for i, line in enumerate(lines): - if line.startswith("from src.app") and "import" in line: - import_insert_idx = i + 1 - if import_insert_idx > 0: - lines.insert(import_insert_idx, import_line) - - # Add router registration - registration_line = f'app.include_router({entity_name.lower()}_router, prefix="/api/v1/{entity_name.lower()}s", tags=["{entity_name.lower()}s"])' - - # Find where to add the registration - for i, line in enumerate(lines): - if "app.include_router" in line and "your_router" in line: - lines.insert(i, registration_line) - break - else: - # Find the last router registration - register_insert_idx = -1 - for i, line in enumerate(lines): - if "app.include_router" in line and "your_router" not in line: - register_insert_idx = i + 1 - - if register_insert_idx > 0: - lines.insert(register_insert_idx, registration_line) - - # Write back - app_file.write_text("\n".join(lines)) - - @entity_app.command() def add( entity_name: str = typer.Argument(None, help="Name of the entity to add"), @@ -197,11 +48,9 @@ def add( - API router with OpenAPI documentation - Comprehensive test coverage """ - # Prompt for entity name if not provided if not entity_name: entity_name = Prompt.ask("[cyan]Entity name") - # Sanitize the entity name entity_name = sanitize_entity_name(entity_name) console.print( @@ -211,7 +60,6 @@ def add( ) ) - # Check if entity already exists project_root = get_project_root() service_entities_dir = project_root / "src" / "app" / "entities" / "service" entity_package_path = service_entities_dir / entity_name.lower() @@ -222,7 +70,6 @@ def add( ) raise typer.Exit(1) - # Prompt for entity fields fields = prompt_for_fields() if not fields: @@ -233,18 +80,14 @@ def add( console.print(f"\n[blue]Creating entity structure for: {entity_name}[/blue]") try: - # Create entity package directory entity_package_path.mkdir(parents=True, exist_ok=True) - # Create entity files console.print("[blue]📄 Creating entity files...[/blue]") create_entity_files(entity_name, fields, entity_package_path) - # Create CRUD router console.print("[blue]🔌 Creating API router...[/blue]") create_crud_router(entity_name, fields) - # Register router with FastAPI app console.print("[blue]📝 Registering router with FastAPI app...[/blue]") register_router_with_app(entity_name) @@ -277,7 +120,6 @@ def add( except Exception as e: console.print(f"[red]❌ Error creating entity: {e}[/red]") - # Clean up on error if entity_package_path.exists(): import shutil @@ -302,7 +144,6 @@ def rm( This operation will ask for confirmation before removing files. """ - # Sanitize entity name entity_name = sanitize_entity_name(entity_name) console.print( @@ -315,7 +156,6 @@ def rm( try: project_root = get_project_root() - # Check if entity exists entity_package_path = ( project_root / "src" / "app" / "entities" / "service" / entity_name.lower() ) @@ -334,10 +174,8 @@ def rm( console.print(f"[red]❌ Entity '{entity_name}' does not exist[/red]") raise typer.Exit(1) - # List files that will be removed files_to_remove = [] - # Entity package files if entity_package_path.exists(): files_to_remove.extend( [ @@ -345,15 +183,13 @@ def rm( str(entity_package_path / "table.py"), str(entity_package_path / "repository.py"), str(entity_package_path / "__init__.py"), - str(entity_package_path), # Directory itself + str(entity_package_path), ] ) - # Router file if router_file.exists(): files_to_remove.append(str(router_file)) - # Show what will be removed console.print("\n[yellow]📂 Files and directories to be removed:[/yellow]") for file_path in files_to_remove: if Path(file_path).is_dir(): @@ -361,7 +197,6 @@ def rm( else: console.print(f" 📄 {file_path}") - # Confirmation prompt (unless --force is used) if not force: console.print("\n[red bold]⚠️ This action cannot be undone![/red bold]") confirm = typer.confirm("Are you sure you want to remove this entity?") @@ -369,7 +204,6 @@ def rm( console.print("[blue]Operation cancelled.[/blue]") return - # Remove entity package directory console.print("\n[blue]🗑️ Removing entity files...[/blue]") if entity_package_path.exists(): import shutil @@ -377,16 +211,13 @@ def rm( shutil.rmtree(entity_package_path) console.print(f" ✅ Removed entity package: {entity_package_path}") - # Remove router file if router_file.exists(): router_file.unlink() console.print(f" ✅ Removed router file: {router_file}") - # Remove router registration from app.py console.print("[blue]📝 Updating FastAPI app registration...[/blue]") unregister_router_from_app(entity_name) - # Success message console.print( f"\n[green]✅ Entity '{entity_name}' removed successfully![/green]" ) @@ -408,54 +239,6 @@ def rm( raise typer.Exit(1) from e -def unregister_router_from_app(entity_name: str) -> None: - """Remove router import and registration from app.py.""" - project_root = get_project_root() - app_file = project_root / "src" / "app" / "api" / "http" / "app.py" - - if not app_file.exists(): - console.print( - "[yellow]⚠️ app.py not found, skipping router unregistration[/yellow]" - ) - return - - content = app_file.read_text() - lines = content.split("\n") - new_lines = [] - - import_pattern = f"from src.app.api.http.routers.service.{entity_name.lower()} import router as {entity_name.lower()}_router" - include_pattern = f'app.include_router({entity_name.lower()}_router, prefix="/api/v1/{entity_name.lower()}s", tags=["{entity_name.lower()}s"])' - - import_found = False - include_found = False - - for line in lines: - # Skip the import line for this entity's router - if import_pattern in line: - console.print(f" ✅ Removed import: {line.strip()}") - import_found = True - continue - - # Skip the include_router line for this entity - if include_pattern in line: - console.print(f" ✅ Removed registration: {line.strip()}") - include_found = True - continue - - new_lines.append(line) - - if not import_found: - console.print("[yellow]⚠️ Import pattern not found in app.py[/yellow]") - if not include_found: - console.print("[yellow]⚠️ Include pattern not found in app.py[/yellow]") - - # Only write back if changes were made - if import_found or include_found: - app_file.write_text("\n".join(new_lines)) - else: - console.print("[yellow]⚠️ No changes made to app.py[/yellow]") - - @entity_app.command() def ls() -> None: """ @@ -488,14 +271,12 @@ def ls() -> None: and not item.name.startswith("_") and item.name != "__pycache__" ): - entity_name = item.name.title() # Convert to title case + entity_name = item.name.title() - # Check for files has_entity = "✅" if (item / "entity.py").exists() else "❌" has_table = "✅" if (item / "table.py").exists() else "❌" has_repository = "✅" if (item / "repository.py").exists() else "❌" - # Check for router router_file = ( project_root / "src" @@ -508,7 +289,6 @@ def ls() -> None: ) has_router = "✅" if router_file.exists() else "❌" - # Check for tests (placeholder for now) has_tests = "❓" # TODO: Implement test detection entities.append( @@ -529,7 +309,6 @@ def ls() -> None: ) return - # Create table table = Table(show_header=True, header_style="bold blue") table.add_column("Entity", style="cyan", no_wrap=True) table.add_column("Entity", style="green", justify="center") diff --git a/src/cli/commands/entity/scaffold.py b/src/cli/commands/entity/scaffold.py new file mode 100644 index 0000000..deba0ed --- /dev/null +++ b/src/cli/commands/entity/scaffold.py @@ -0,0 +1,198 @@ +"""Entity scaffolding utilities.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from rich.prompt import Prompt + +from src.cli.shared.console import console +from src.utils.paths import get_project_root + +from .templates import render_template_to_file + + +def sanitize_entity_name(name: str) -> str: + """Sanitize entity name to conform to Python naming conventions.""" + words = re.findall(r"[a-zA-Z0-9]+", name) + return "".join(word.capitalize() for word in words) + + +def sanitize_field_name(name: str) -> str: + """Sanitize field name to conform to Python snake_case conventions.""" + words = re.findall(r"[a-zA-Z0-9]+", name) + return "_".join(word.lower() for word in words) + + +def prompt_for_fields() -> list[dict[str, str | bool]]: + """Prompt user for entity fields.""" + fields: list[dict[str, str | bool]] = [] + console.print( + "\n[blue]Define entity fields (press Enter without a name to finish):[/blue]" + ) + + while True: + field_name = Prompt.ask("[cyan]Field name", default="") + if not field_name.strip(): + break + + field_name = sanitize_field_name(field_name) + + field_type = Prompt.ask( + f"[cyan]Type for '{field_name}'", + choices=["str", "int", "float", "bool", "datetime"], + default="str", + ) + + optional = ( + Prompt.ask( + f"[cyan]Is '{field_name}' optional?", choices=["y", "n"], default="n" + ) + == "y" + ) + + description = Prompt.ask( + f"[cyan]Description for '{field_name}'", + default=f"{field_name.replace('_', ' ').title()}", + ) + + fields.append( + { + "name": field_name, + "type": field_type, + "optional": optional, + "description": description, + } + ) + + console.print(f"[green]✓[/green] Added field: {field_name}: {field_type}") + + return fields + + +def create_entity_files( + entity_name: str, fields: list[dict[str, str | bool]], package_path: Path +) -> None: + """Create all entity files using Jinja2 templates.""" + context = {"entity_name": entity_name, "fields": fields} + + render_template_to_file("entity.py.j2", package_path / "entity.py", context) + render_template_to_file("table.py.j2", package_path / "table.py", context) + render_template_to_file("repository.py.j2", package_path / "repository.py", context) + render_template_to_file("__init__.py.j2", package_path / "__init__.py", context) + + +def create_crud_router(entity_name: str, fields: list[dict[str, str | bool]]) -> None: + """Create a CRUD router for the entity using templates.""" + router_dir = ( + get_project_root() / "src" / "app" / "api" / "http" / "routers" / "service" + ) + router_dir.mkdir(parents=True, exist_ok=True) + + router_file = router_dir / f"{entity_name.lower()}.py" + context = {"entity_name": entity_name, "fields": fields} + render_template_to_file("router.py.j2", router_file, context) + + routers_init = router_dir / "__init__.py" + if not routers_init.exists(): + routers_init.write_text('"""Service routers package."""\n') + + +def register_router_with_app(entity_name: str) -> None: + """Add import and registration for the new router in app.py.""" + app_file = get_project_root() / "src" / "app" / "api" / "http" / "app.py" + + content = app_file.read_text() + + import_line = ( + "from src.app.api.http.routers.service." + f"{entity_name.lower()} import router as {entity_name.lower()}_router" + ) + + lines = content.split("\n") + import_insert_idx = -1 + + for i, line in enumerate(lines): + if "from src.app.api.http.routers" in line and "import router" in line: + import_insert_idx = i + 1 + + if import_insert_idx > 0: + lines.insert(import_insert_idx, import_line) + else: + for i, line in enumerate(lines): + if line.startswith("from src.app") and "import" in line: + import_insert_idx = i + 1 + if import_insert_idx > 0: + lines.insert(import_insert_idx, import_line) + + registration_line = ( + f"app.include_router({entity_name.lower()}_router, " + f'prefix="/api/v1/{entity_name.lower()}s", tags=["{entity_name.lower()}s"])' + ) + + for i, line in enumerate(lines): + if "app.include_router" in line and "your_router" in line: + lines.insert(i, registration_line) + break + else: + register_insert_idx = -1 + for i, line in enumerate(lines): + if "app.include_router" in line and "your_router" not in line: + register_insert_idx = i + 1 + + if register_insert_idx > 0: + lines.insert(register_insert_idx, registration_line) + + app_file.write_text("\n".join(lines)) + + +def unregister_router_from_app(entity_name: str) -> None: + """Remove router import and registration from app.py.""" + project_root = get_project_root() + app_file = project_root / "src" / "app" / "api" / "http" / "app.py" + + if not app_file.exists(): + console.print( + "[yellow]⚠️ app.py not found, skipping router unregistration[/yellow]" + ) + return + + content = app_file.read_text() + lines = content.split("\n") + new_lines = [] + + import_pattern = ( + "from src.app.api.http.routers.service." + f"{entity_name.lower()} import router as {entity_name.lower()}_router" + ) + include_pattern = ( + f"app.include_router({entity_name.lower()}_router, " + f'prefix="/api/v1/{entity_name.lower()}s", tags=["{entity_name.lower()}s"])' + ) + + import_found = False + include_found = False + + for line in lines: + if import_pattern in line: + console.print(f" ✅ Removed import: {line.strip()}") + import_found = True + continue + + if include_pattern in line: + console.print(f" ✅ Removed registration: {line.strip()}") + include_found = True + continue + + new_lines.append(line) + + if not import_found: + console.print("[yellow]⚠️ Import pattern not found in app.py[/yellow]") + if not include_found: + console.print("[yellow]⚠️ Include pattern not found in app.py[/yellow]") + + if import_found or include_found: + app_file.write_text("\n".join(new_lines)) + else: + console.print("[yellow]⚠️ No changes made to app.py[/yellow]") diff --git a/src/cli/commands/entity/templates.py b/src/cli/commands/entity/templates.py new file mode 100644 index 0000000..fc43ea0 --- /dev/null +++ b/src/cli/commands/entity/templates.py @@ -0,0 +1,26 @@ +"""Jinja2 templates for entity scaffolding.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from jinja2 import Environment, FileSystemLoader + +from src.utils.paths import get_project_root + + +def get_template_env() -> Environment: + """Get Jinja2 environment for template rendering.""" + template_dir = get_project_root() / "src" / "cli" / "templates" + return Environment(loader=FileSystemLoader(Path(template_dir))) + + +def render_template_to_file( + template_name: str, output_path: Path, context: dict[str, Any] +) -> None: + """Render a Jinja2 template to a file.""" + env = get_template_env() + template = env.get_template(template_name) + content = template.render(**context) + output_path.write_text(content) diff --git a/src/cli/commands/fly.py b/src/cli/commands/fly.py index 666a913..a761a61 100644 --- a/src/cli/commands/fly.py +++ b/src/cli/commands/fly.py @@ -10,7 +10,7 @@ import typer -from .shared import console, print_header +from src.cli.shared.console import console # --------------------------------------------------------------------------- # Typer App @@ -46,7 +46,7 @@ def up( See docs/fastapi-flyio-kubernetes.md for details. """ - print_header("Fly.io Kubernetes Deployment") + console.print_header("Fly.io Kubernetes Deployment") _show_coming_soon_message() @@ -65,7 +65,7 @@ def down( This command is a placeholder for future FKS deployment support. """ - print_header("Removing Fly.io Deployment") + console.print_header("Removing Fly.io Deployment") _show_coming_soon_message() @@ -84,7 +84,7 @@ def status( This command is a placeholder for future FKS deployment support. """ - print_header("Fly.io Deployment Status") + console.print_header("Fly.io Deployment Status") _show_coming_soon_message() @@ -94,7 +94,7 @@ def clusters() -> None: This command is a placeholder for future FKS support. """ - print_header("FKS Clusters") + console.print_header("FKS Clusters") _show_coming_soon_message() diff --git a/src/cli/commands/k8s.py b/src/cli/commands/k8s.py index ce40909..7b249d5 100644 --- a/src/cli/commands/k8s.py +++ b/src/cli/commands/k8s.py @@ -4,48 +4,18 @@ Kubernetes deployments via Helm. """ -from typing import TYPE_CHECKING, Annotated +from typing import Annotated import typer from rich.panel import Panel from rich.table import Table -from src.infra.k8s import Kr8sController, run_sync - -from .shared import ( - confirm_action, - console, - get_project_root, - print_header, - with_error_handling, -) - -if TYPE_CHECKING: - from src.cli.deployment.helm_deployer.deployer import HelmDeployer - - -# --------------------------------------------------------------------------- -# Kubernetes Controller (module-level singleton) -# --------------------------------------------------------------------------- - -_controller = Kr8sController() - - -# --------------------------------------------------------------------------- -# Deployer Factory -# --------------------------------------------------------------------------- - - -def _get_deployer() -> "HelmDeployer": - """Get the Helm deployer instance. - - Returns: - HelmDeployer instance configured for current project - """ - from src.cli.deployment.helm_deployer.deployer import HelmDeployer - - return HelmDeployer(console, get_project_root()) +from src.cli.context import get_cli_context +from src.cli.deployment.helm_deployer.deployer import get_deployer +from src.cli.shared.console import console, with_error_handling +from src.utils.paths import get_project_root +from .k8s_db import k8s_db_app # --------------------------------------------------------------------------- # Helper Functions @@ -61,8 +31,9 @@ def _check_cluster_issuer_ready(issuer_name: str) -> bool: Returns: True if the ClusterIssuer exists and is ready, False otherwise """ - status = run_sync(_controller.get_cluster_issuer_status(issuer_name)) - return status.exists and status.ready + controller = get_cli_context().k8s_controller + status = controller.get_cluster_issuer_status(issuer_name) + return bool(status.exists and status.ready) def _check_cert_manager_installed() -> bool: @@ -71,7 +42,9 @@ def _check_cert_manager_installed() -> bool: Returns: True if cert-manager pods are running, False otherwise """ - return run_sync(_controller.check_cert_manager_installed()) + controller = get_cli_context().k8s_controller + result: bool = controller.check_cert_manager_installed() + return result def _install_cert_manager() -> bool: @@ -149,7 +122,8 @@ def _wait_for_cluster_issuer(issuer_name: str, timeout: int = 60) -> bool: time.sleep(2) # Check if it exists but isn't ready - yaml_output = run_sync(_controller.get_cluster_issuer_yaml(issuer_name)) + controller = get_cli_context().k8s_controller + yaml_output = controller.get_cluster_issuer_yaml(issuer_name) if yaml_output: console.print("[yellow]ClusterIssuer exists but not ready yet[/yellow]") console.print(f"[dim]{yaml_output}[/dim]") @@ -227,6 +201,13 @@ def up( help="Use Let's Encrypt staging (with --ingress-tls-auto)", ), ] = False, + skip_db_check: Annotated[ + bool, + typer.Option( + "--skip-db-check", + help="Skip PostgreSQL verification before deployment", + ), + ] = False, ) -> None: """Deploy to Kubernetes cluster using Helm. @@ -235,6 +216,8 @@ def up( - Builds Docker images with content-based tagging - Loads images into target cluster (Minikube, Kind, or registry) - Deploys Kubernetes secrets + - Restarts postgres StatefulSet (if bundled postgres enabled) + - Verifies PostgreSQL is accessible (unless --skip-db-check) - Syncs config.yaml to Helm values - Deploys via Helm upgrade --install - Waits for rollouts to complete @@ -245,8 +228,9 @@ def up( uv run api-forge-cli k8s up --registry ghcr.io/myuser uv run api-forge-cli k8s up --ingress --ingress-host api.example.com uv run api-forge-cli k8s up --ingress --ingress-host api.example.com --ingress-tls-auto + uv run api-forge-cli k8s up --skip-db-check # Skip database verification """ - print_header("Deploying to Kubernetes") + console.print_header("Deploying to Kubernetes") # Validate TLS options if ingress_tls_auto and ingress_tls_secret: @@ -256,13 +240,11 @@ def up( raise typer.Exit(1) if ingress_tls_auto and not ingress: - console.print( - "[yellow]--ingress-tls-auto implies --ingress, enabling it[/yellow]" - ) + console.info("--ingress-tls-auto implies --ingress, enabling it") ingress = True if ingress_tls_staging and not ingress_tls_auto: - console.print("[red]--ingress-tls-staging requires --ingress-tls-auto[/red]") + console.error("--ingress-tls-staging requires --ingress-tls-auto") raise typer.Exit(1) # Check cert-manager is ready if using auto TLS @@ -280,8 +262,7 @@ def up( f" [cyan]uv run api-forge-cli k8s setup-tls --email your@email.com{staging_flag}[/cyan]" ) raise typer.Exit(1) - - deployer = _get_deployer() + deployer = get_deployer() deployer.deploy( namespace=namespace, registry=registry, @@ -290,6 +271,7 @@ def up( ingress_tls_secret=ingress_tls_secret, ingress_tls_auto=ingress_tls_auto, ingress_tls_staging=ingress_tls_staging, + skip_db_check=skip_db_check, ) @@ -322,10 +304,10 @@ def down( uv run api-forge-cli k8s down -n my-namespace uv run api-forge-cli k8s down -y # Skip confirmation """ - print_header("Removing Kubernetes Deployment") + console.print_header("Removing Kubernetes Deployment") if not yes: - if not confirm_action( + if not console.confirm_action( "Remove Kubernetes deployment", f"This will:\n" f" • Uninstall the Helm release\n" @@ -335,7 +317,7 @@ def down( console.print("[dim]Operation cancelled[/dim]") raise typer.Exit(0) - deployer = _get_deployer() + deployer = get_deployer() deployer.teardown(namespace=namespace) @@ -359,9 +341,9 @@ def status( uv run api-forge-cli k8s status uv run api-forge-cli k8s status -n my-namespace """ - print_header("Kubernetes Deployment Status") + console.print_header("Kubernetes Deployment Status") - deployer = _get_deployer() + deployer = get_deployer() deployer.show_status(namespace=namespace) @@ -395,9 +377,9 @@ def history( uv run api-forge-cli k8s history uv run api-forge-cli k8s history --max 5 """ - print_header("Release History") + console.print_header("Release History") - deployer = _get_deployer() + deployer = get_deployer() # Get release history history_data = deployer.commands.helm.history( @@ -483,9 +465,9 @@ def rollback( uv run api-forge-cli k8s rollback 3 # Specific revision uv run api-forge-cli k8s history # View history first """ - print_header("Rollback Deployment") + console.print_header("Rollback Deployment") - deployer = _get_deployer() + deployer = get_deployer() # Get release history history_data = deployer.commands.helm.history( @@ -493,9 +475,9 @@ def rollback( ) if not history_data: - console.print( - f"[red]No release history found for '{deployer.constants.HELM_RELEASE_NAME}' " - f"in namespace '{namespace}'[/red]" + console.error( + f"No release history found for '{deployer.constants.HELM_RELEASE_NAME}' " + f"in namespace '{namespace}'" ) console.print("\n[dim]Make sure the release exists and you have access.[/dim]") raise typer.Exit(1) @@ -505,18 +487,16 @@ def rollback( current_revision = int(current.get("revision", 0)) if current_revision <= 1: - console.print( - "[yellow]⚠ Only one revision exists. Nothing to rollback to.[/yellow]" - ) + console.warn("Only one revision exists. Nothing to rollback to.") raise typer.Exit(0) # Determine target revision target_revision = revision if revision is not None else current_revision - 1 if target_revision < 1 or target_revision >= current_revision: - console.print( - f"[red]Invalid revision {target_revision}. " - f"Must be between 1 and {current_revision - 1}.[/red]" + console.error( + f"Invalid revision {target_revision}. " + f"Must be between 1 and {current_revision - 1}." ) raise typer.Exit(1) @@ -553,7 +533,7 @@ def rollback( # Confirm if not yes: - if not confirm_action( + if not console.confirm_action( f"Rollback to revision {target_revision}", f"This will restore the deployment in namespace '{namespace}' " f"to revision {target_revision}.\n" @@ -579,12 +559,10 @@ def rollback( ) if result.success: - console.print( - f"\n[bold green]✅ Successfully rolled back to revision {target_revision}![/bold green]" - ) + console.ok(f"\nSuccessfully rolled back to revision {target_revision}!") console.print("\n[dim]Run 'uv run api-forge-cli k8s status' to verify.[/dim]") else: - console.print("\n[bold red]❌ Rollback failed[/bold red]") + console.error("\nRollback failed") if result.stderr: console.print(Panel(result.stderr, title="Error", border_style="red")) raise typer.Exit(1) @@ -656,17 +634,17 @@ def logs( label_selector = "app=api-forge" if not pod else None try: - result = run_sync( - _controller.get_pod_logs( - namespace=namespace, - pod=pod, - container=container, - label_selector=label_selector, - follow=follow, - tail=tail, - previous=previous, - ) + controller = get_cli_context().k8s_controller + result = controller.get_pod_logs( + namespace=namespace, + pod=pod, + container=container, + label_selector=label_selector, + follow=follow, + tail=tail, + previous=previous, ) + if result.stdout: console.print(result.stdout) if not result.success and result.stderr: @@ -715,7 +693,7 @@ def setup_tls( uv run api-forge-cli k8s setup-tls --email admin@example.com --staging uv run api-forge-cli k8s up --ingress --ingress-host api.example.com --ingress-tls-auto """ - print_header("TLS Setup with cert-manager") + console.print_header("TLS Setup with cert-manager") if not email: console.print("[red]Email is required for Let's Encrypt registration.[/red]") @@ -729,14 +707,14 @@ def setup_tls( console.print("\n[bold]Step 1/3:[/bold] Checking cert-manager installation...") if _check_cert_manager_installed(): - console.print("[green]✓[/green] cert-manager is already installed") + console.ok("cert-manager is already installed") else: if install_cert_manager: - console.print("[yellow]cert-manager not found, installing...[/yellow]") + console.info("cert-manager not found, installing...") if not _install_cert_manager(): raise typer.Exit(1) else: - console.print("[red]cert-manager is not installed.[/red]") + console.error("cert-manager is not installed.") console.print( "\n[dim]Run with --install-cert-manager or install manually:[/dim]" ) @@ -752,19 +730,15 @@ def setup_tls( if staging: server = "https://acme-staging-v02.api.letsencrypt.org/directory" issuer_name = "letsencrypt-staging" - console.print( - "[yellow]Using Let's Encrypt staging server (for testing)[/yellow]" - ) + console.info("Using Let's Encrypt staging server (for testing)") else: server = "https://acme-v02.api.letsencrypt.org/directory" issuer_name = "letsencrypt-prod" - console.print("[cyan]Using Let's Encrypt production server[/cyan]") + console.info("Using Let's Encrypt production server") # Check if issuer already exists and is ready if _check_cluster_issuer_ready(issuer_name): - console.print( - f"[green]✓[/green] ClusterIssuer '{issuer_name}' already exists and is ready" - ) + console.ok(f"ClusterIssuer '{issuer_name}' already exists and is ready") else: # Create ClusterIssuer manifest file (version-controlled, GitOps-friendly) project_root = get_project_root() @@ -808,32 +782,31 @@ def setup_tls( # Apply the manifest console.print(f"[dim]Applying ClusterIssuer '{issuer_name}'...[/dim]") - result = run_sync(_controller.apply_manifest(issuer_file)) + controller = get_cli_context().k8s_controller + result = controller.apply_manifest(issuer_file) if not result.success: - console.print("[red]Failed to create ClusterIssuer[/red]") + console.error("Failed to create ClusterIssuer") if result.stderr: console.print(Panel(result.stderr, title="Error", border_style="red")) raise typer.Exit(1) - console.print(f"[green]✓[/green] ClusterIssuer '{issuer_name}' created") + console.ok(f"ClusterIssuer '{issuer_name}' created") # Step 3: Wait for ClusterIssuer to be ready console.print("\n[bold]Step 3/3:[/bold] Waiting for ClusterIssuer to be ready...") if _wait_for_cluster_issuer(issuer_name, timeout=60): - console.print(f"[green]✓[/green] ClusterIssuer '{issuer_name}' is ready") + console.ok(f"ClusterIssuer '{issuer_name}' is ready") else: - console.print( - f"[yellow]⚠ ClusterIssuer '{issuer_name}' created but not ready yet[/yellow]" - ) + console.warn(f"ClusterIssuer '{issuer_name}' created but not ready yet") console.print( "[dim]This is normal - it will become ready when you create your first certificate.[/dim]" ) # Success message with next steps console.print("\n" + "=" * 60) - console.print("[bold green]✅ TLS setup complete![/bold green]") + console.ok("TLS setup complete!") console.print("=" * 60) console.print("\n[bold cyan]Deploy with automatic TLS:[/bold cyan]") @@ -851,15 +824,20 @@ def setup_tls( console.print(" 6. cert-manager auto-renews before expiry") if staging: - console.print( - "\n[yellow]⚠ Staging certificates are not trusted by browsers.[/yellow]" - ) - console.print( - "[yellow] Run without --staging for production certificates.[/yellow]" + console.warn( + "Staging certificates are not trusted by browsers. Use only for testing." ) + console.warn(" Run without --staging for production certificates.") console.print("\n[bold cyan]Manifest saved to:[/bold cyan]") console.print(f" [dim]infra/helm/cert-manager/{issuer_name}.yaml[/dim]") console.print( " [dim]Commit this file to version control for GitOps workflows.[/dim]" ) + + +# --------------------------------------------------------------------------- +# Register Subcommands +# --------------------------------------------------------------------------- + +k8s_app.add_typer(k8s_db_app, name="db") diff --git a/src/cli/commands/k8s_db.py b/src/cli/commands/k8s_db.py new file mode 100644 index 0000000..4341839 --- /dev/null +++ b/src/cli/commands/k8s_db.py @@ -0,0 +1,884 @@ +"""PostgreSQL database management for Kubernetes deployments. + +This module provides db subcommands under 'k8s' for managing PostgreSQL +databases in Kubernetes/Helm environments. +""" + +import subprocess +from pathlib import Path +from typing import Annotated + +import typer + +from src.cli.commands.db import ( + DbRuntime, + get_k8s_runtime, + run_backup, + run_init, + run_migrate, + run_reset, + run_status, + run_sync, + run_verify, +) +from src.cli.commands.db_utils import ( + configure_external_database, + read_env_example_values, + update_bundled_postgres_config, + update_env_file, +) +from src.cli.context import get_cli_context +from src.cli.deployment.helm_deployer import ConfigSynchronizer +from src.cli.deployment.status_display import is_temporal_enabled +from src.cli.shared.console import console, with_error_handling +from src.infra.constants import DeploymentConstants, DeploymentPaths +from src.infra.k8s import get_namespace, get_postgres_label +from src.infra.k8s.controller import KubernetesControllerSync +from src.utils.paths import get_project_root + +# --------------------------------------------------------------------------- +# Typer App +# --------------------------------------------------------------------------- + +k8s_db_app = typer.Typer( + name="db", + help="PostgreSQL database management for Kubernetes.", + no_args_is_help=True, +) + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + + +def _get_components() -> tuple[ + DeploymentConstants, DeploymentPaths, KubernetesControllerSync +]: + """Return deployment constants, paths, and controller from CLI context.""" + ctx = get_cli_context() + return ctx.constants, ctx.paths, ctx.k8s_controller + + +def _get_namespace_and_label() -> tuple[str, str]: + """Resolve namespace and postgres label at call time.""" + return get_namespace(), get_postgres_label() + + +def _get_runtime() -> DbRuntime: + """Return the Kubernetes DB runtime.""" + return get_k8s_runtime() + + +@k8s_db_app.command() +@with_error_handling +def create( + # Mode selection (mutually exclusive) + bundled: Annotated[ + bool, + typer.Option( + "--bundled", + help="Deploy bundled PostgreSQL to Kubernetes", + ), + ] = False, + external: Annotated[ + bool, + typer.Option( + "--external", + help="Configure connection to external PostgreSQL", + ), + ] = False, + # External mode parameters + connection_string: Annotated[ + str | None, + typer.Option( + "--connection-string", + "-c", + help="Full PostgreSQL connection string (postgres://user:pass@host:port/db)", + ), + ] = None, + host: Annotated[ + str | None, + typer.Option( + "--host", + "-H", + help="Database host (overrides connection string)", + ), + ] = None, + port: Annotated[ + str | None, + typer.Option( + "--port", + "-P", + help="Database port (overrides connection string)", + ), + ] = None, + username: Annotated[ + str | None, + typer.Option( + "--username", + "-u", + help="Database username (overrides connection string)", + ), + ] = None, + password: Annotated[ + str | None, + typer.Option( + "--password", + "-p", + help="Database password (overrides connection string)", + ), + ] = None, + database: Annotated[ + str | None, + typer.Option( + "--database", + "-d", + help="Database name (overrides connection string)", + ), + ] = None, + sslmode: Annotated[ + str | None, + typer.Option( + "--sslmode", + help="SSL mode (e.g., require, verify-full)", + ), + ] = None, + tls_ca: Annotated[ + str | None, + typer.Option( + "--tls-ca", + help="Path to TLS CA certificate file for external PostgreSQL (e.g., Aiven CA)", + ), + ] = None, + # Bundled mode parameters + values_file: Annotated[ + Path | None, + typer.Option( + "--values", + "-f", + help="Custom Helm values file (bundled mode only)", + ), + ] = None, + wait: Annotated[ + bool, + typer.Option( + "--wait/--no-wait", + help="Wait for PostgreSQL to be ready (bundled mode only)", + ), + ] = True, +) -> None: + """Configure PostgreSQL database for Kubernetes deployment. + + Two modes are available: + + --bundled: Deploy a bundled PostgreSQL instance to Kubernetes using Helm. + This deploys secrets, the PostgreSQL chart, and configures security settings. + + --external: Configure connection to an external PostgreSQL instance (e.g., Aiven, RDS). + This updates .env and config.yaml with connection details. + + Examples: + # Deploy bundled PostgreSQL + uv run api-forge-cli k8s db create --bundled + uv run api-forge-cli k8s db create --bundled --values custom-values.yaml + + # Configure external PostgreSQL with connection string + uv run api-forge-cli k8s db create --external \\ + --connection-string "postgres://admin:secret@db.example.com:5432/mydb?sslmode=require" + + # Configure external PostgreSQL with individual parameters + uv run api-forge-cli k8s db create --external \\ + --host db.example.com --port 5432 \\ + --username admin --password secret --database mydb + + # Mix: connection string with override + uv run api-forge-cli k8s db create --external \\ + --connection-string "postgres://user:pass@host:5432/db" \\ + --password new-secret # Override password from connection string + """ + # Validate mode selection + if bundled and external: + console.print("[red]❌ Cannot use both --bundled and --external[/red]") + raise typer.Exit(1) + + if not bundled and not external: + console.print("[red]❌ Must specify either --bundled or --external[/red]") + console.print("[dim]Use --bundled to deploy PostgreSQL to Kubernetes[/dim]") + console.print("[dim]Use --external to configure an external database[/dim]") + raise typer.Exit(1) + + if external: + configure_external_database( + connection_string=connection_string, + host=host, + port=port, + username=username, + password=password, + database=database, + sslmode=sslmode, + tls_ca=tls_ca, + next_steps_cmd_prefix="k8s", + ) + else: + _create_bundled(values_file=values_file, wait=wait) + + +def _create_bundled(*, values_file: Path | None, wait: bool) -> None: + """Deploy bundled PostgreSQL to Kubernetes using Helm.""" + from rich.progress import Progress + + from src.cli.deployment.helm_deployer.secret_manager import SecretManager + from src.cli.deployment.shell_commands import ShellCommands + + constants, paths, controller = _get_components() + namespace, postgres_label = _get_namespace_and_label() + + console.print_header("Deploying Bundled PostgreSQL to Kubernetes") + + # Read bundled defaults from .env.example + bundled_keys = ["PRODUCTION_DATABASE_URL", "PG_SUPERUSER", "PG_DB"] + bundled_defaults = read_env_example_values(bundled_keys) + + if not bundled_defaults: + console.print("[red]❌ Could not read bundled defaults from .env.example[/red]") + raise typer.Exit(1) + + # Update .env with bundled defaults from .env.example + console.info("Updating .env file for bundled PostgreSQL...") + update_env_file(bundled_defaults) + console.ok(".env file updated") + + # Update config.yaml + console.info("Updating config.yaml...") + update_bundled_postgres_config(enabled=True) + console.ok("config.yaml updated (bundled_postgres.enabled=true)") + project_root = get_project_root() + + # Step 1: Deploy secrets required by PostgreSQL + console.info("Deploying secrets...") + commands = ShellCommands(project_root) + secret_manager = SecretManager( + commands=commands, + console=console, + paths=paths, + ) + secret_manager.deploy_secrets( + namespace=namespace, + progress_factory=Progress, + ) + console.ok("Secrets deployed") + + # Step 1.5: Copy config files + console.info("Copying config files to Helm staging area...") + sync = ConfigSynchronizer( + console=console, + paths=paths, + ) + sync.copy_config_files(progress_factory=Progress) + console.ok("Config files copied") + + # Step 2: Deploy PostgreSQL + # Try standalone bundled chart first, fallback to Bitnami + standalone_chart_path = paths.postgres_standalone_chart + + if ( + standalone_chart_path.exists() + and (standalone_chart_path / "Chart.yaml").exists() + ): + chart = str(standalone_chart_path) + console.info(f"Using bundled PostgreSQL chart: {standalone_chart_path.name}") + if not values_file: + # Use the standalone chart's values.yaml + standalone_values = standalone_chart_path / "values.yaml" + if standalone_values.exists(): + values_file = standalone_values + console.info(f"Using chart values: {standalone_values.name}") + else: + # Fallback to Bitnami chart + chart = "oci://registry-1.docker.io/bitnamicharts/postgresql" + console.warn("Bundled chart not found, using Bitnami PostgreSQL chart") + console.print( + "[dim]Note: Bitnami is deprecating free charts. " + "Consider using bundled chart or managed database.[/dim]" + ) + if not values_file: + values_file = paths.bitnami_postgres_values_yaml + + if values_file and values_file.exists(): + console.info(f"Using values file: {values_file}") + else: + console.info("No values file provided or found, using defaults") + values_file = None + + # Pre-flight check: Check if StatefulSet exists and might have conflicts + # This prevents Helm from hanging on --wait when immutable fields changed + if controller.resource_exists( + "statefulset", constants.POSTGRES_RESOURCE_NAME, namespace + ): + console.warn(f"StatefulSet '{constants.POSTGRES_RESOURCE_NAME}' already exists") + console.print( + "[yellow]⚠[/yellow] This may cause conflicts if template changed immutable fields" + ) + action = console.prompt_resource_conflict( + resource_type="StatefulSet", + resource_name=constants.POSTGRES_RESOURCE_NAME, + namespace=namespace, + data_warning=True, + ) + + if action == "abort": + console.error("Deployment aborted by user") + raise typer.Exit(1) + + if action == "recreate": + console.warn("Recreating PostgreSQL StatefulSet...") + console.print("[dim]Note: PVCs will be retained to preserve data[/dim]") + console.print("[dim]Note: Helm release history will be reset[/dim]") + + # Step 1: Delete the StatefulSet by name (keeps PVCs with orphan cascade) + # Using delete_resource by name because the deployed resource may have + # different labels than the new template (this is why we're in this situation) + delete_result = controller.delete_resource( + "statefulset", + constants.POSTGRES_RESOURCE_NAME, + namespace, + cascade="orphan", + wait=True, + ) + + if not delete_result.success: + console.error(f"Failed to delete StatefulSet:\n{delete_result.stderr}") + raise typer.Exit(1) + + # Wait for StatefulSet to actually be deleted + console.info("Waiting for StatefulSet deletion...") + import time + + for _ in range(30): # Wait up to 30 seconds + if not controller.resource_exists( + "statefulset", + constants.POSTGRES_RESOURCE_NAME, + namespace, + ): + break + time.sleep(1) + else: + console.error( + "Timeout waiting for StatefulSet deletion. " + "Please delete it manually and retry." + ) + raise typer.Exit(1) + + console.ok("StatefulSet deleted") + + # Step 1.5: Delete orphaned pods + # The StatefulSet was deleted with cascade=orphan, so pods are still running + # We need to delete them so Helm can create fresh ones + console.info("Deleting orphaned PostgreSQL pods...") + pod_delete_result = controller.delete_resources_by_label( + "pod", + namespace, + constants.POSTGRES_POD_LABEL, + force=False, + cascade=None, + ) + if not pod_delete_result.success: + console.warn( + f"Failed to delete pods (may not exist): {pod_delete_result.stderr}" + ) + else: + console.ok("Orphaned pods deleted") + + # Step 2: Clear Helm's cached manifest state + # This is required because Helm does a 3-way merge and will + # still try to "update" based on its stored old manifest + console.info("Clearing Helm release metadata...") + controller.delete_helm_secrets(namespace, "postgresql") + console.ok("Helm release metadata cleared") + + console.print( + "[green]✓[/green] Ready for fresh installation (PVCs retained)" + ) + + # Build helm command + helm_args = [ + "helm", + "upgrade", + "--install", + "postgresql", + chart, + "-n", + namespace, + "--create-namespace", + ] + + if values_file: + helm_args.extend(["-f", str(values_file)]) + + if wait: + helm_args.append("--wait") + + console.print("[cyan]ℹ[/cyan] Installing PostgreSQL...") + result = subprocess.run(helm_args, capture_output=True, text=True) + + if result.returncode != 0: + # Check if this is a StatefulSet immutable field error + if "spec: Forbidden: updates to statefulset spec" in result.stderr: + # Check if StatefulSet exists using controller + if controller.resource_exists( + "statefulset", constants.POSTGRES_RESOURCE_NAME, namespace + ): + # StatefulSet exists - use the prompt helper + action = console.prompt_resource_conflict( + resource_type="StatefulSet", + resource_name=constants.POSTGRES_RESOURCE_NAME, + namespace=namespace, + data_warning=True, + ) + + if action == "recreate": + console.warn("Recreating PostgreSQL StatefulSet...") + console.print( + "[dim]Note: PVCs will be retained to preserve data[/dim]" + ) + console.print("[dim]Note: Helm release history will be reset[/dim]") + + # Step 1: Delete the StatefulSet by name (keeps PVCs with orphan cascade) + # Using delete_resource by name because the deployed resource may have + # different labels than the new template (this is why we're in this situation) + delete_result = controller.delete_resource( + "statefulset", + constants.POSTGRES_RESOURCE_NAME, + namespace, + cascade="orphan", + wait=True, + ) + + if not delete_result.success: + console.error( + f"Failed to delete StatefulSet:\n{delete_result.stderr}" + ) + raise typer.Exit(1) + + # Wait for StatefulSet to actually be deleted + console.info("Waiting for StatefulSet deletion...") + import time + + for _ in range(30): # Wait up to 30 seconds + if not controller.resource_exists( + "statefulset", + constants.POSTGRES_RESOURCE_NAME, + namespace, + ): + break + time.sleep(1) + else: + console.error( + "Timeout waiting for StatefulSet deletion. " + "Please delete it manually and retry." + ) + raise typer.Exit(1) + + console.ok("StatefulSet deleted") + + # Step 2: Clear Helm's cached manifest state + # This is required because Helm does a 3-way merge and will + # still try to "update" based on its stored old manifest + console.info("Clearing Helm release metadata...") + controller.delete_helm_secrets(namespace, "postgresql") + console.ok("Helm release metadata cleared") + + # Step 3: Fresh Helm install (upgrade --install will install since + # release metadata is gone) + console.info("Installing PostgreSQL...") + retry_result = subprocess.run( + helm_args, capture_output=True, text=True + ) + + if retry_result.returncode != 0: + console.error( + f"Failed to install PostgreSQL:\n{retry_result.stderr}" + ) + raise typer.Exit(1) + + console.ok("PostgreSQL deployed to Kubernetes") + elif action == "skip": + console.warn( + "Skipping PostgreSQL deployment - keeping existing StatefulSet" + ) + console.print("\n[dim]To manually update, run:[/dim]") + console.print( + f"[dim] kubectl delete statefulset {constants.POSTGRES_RESOURCE_NAME} -n {namespace} --cascade=orphan[/dim]" + ) + console.print( + "[dim] uv run api-forge-cli k8s db create --bundled[/dim]" + ) + return + else: # cancel + console.print("[dim]Operation cancelled.[/dim]") + raise typer.Exit(0) + else: + # StatefulSet doesn't exist but we got the error - unexpected + console.error(f"Unexpected error:\n{result.stderr}") + raise typer.Exit(1) + else: + # Different error + console.error(f"Failed to install PostgreSQL:\n{result.stderr}") + raise typer.Exit(1) + + console.ok("PostgreSQL deployed to Kubernetes") + console.print(f"\n[dim]Namespace: {namespace}[/dim]") + + # Auto-initialize database after successful deployment + console.info("\nInitializing database...") + + try: + success = run_init(_get_runtime()) + if not success: + console.error("Database initialization failed") + console.print("\n[dim]You can retry with:[/dim]") + console.print(" uv run api-forge-cli k8s db init") + raise typer.Exit(1) + except Exception as exc: + console.error(f"Database initialization failed: {exc}") + console.print("\n[dim]You can retry with:[/dim]") + console.print(" uv run api-forge-cli k8s db init") + raise typer.Exit(1) from exc + + console.ok("Database initialized successfully") + console.print("\n[dim]Next steps:[/dim]") + console.print(" - Run 'uv run api-forge-cli k8s db verify' to verify setup") + + +@k8s_db_app.command(name="init") +@with_error_handling +def init_db() -> None: + """Initialize the PostgreSQL database with roles and schema. + + This command: + - Creates the owner, app user, and read-only roles + - Creates the application database + - Sets up the schema with proper privileges + - Optionally initializes Temporal databases + + Examples: + uv run api-forge-cli k8s db init + """ + console.print_header("Initializing PostgreSQL Database (Kubernetes)") + success = run_init(_get_runtime()) + + if not success: + raise typer.Exit(1) + + +@k8s_db_app.command() +@with_error_handling +def verify() -> None: + """Verify PostgreSQL database setup and configuration. + + This command checks: + - Pod existence and readiness + - Role existence and attributes + - Database and schema ownership + - Table and sequence privileges + + Examples: + uv run api-forge-cli k8s db verify + """ + console.print_header("Verifying PostgreSQL Configuration (Kubernetes)") + success = run_verify(_get_runtime(), superuser_mode=True) + + if not success: + console.info( + 'Please run "uv run api-forge-cli k8s db init" to re-initialize the database.' + ) + raise typer.Exit(1) + + +@k8s_db_app.command() +@with_error_handling +def sync() -> None: + """Synchronize PostgreSQL role passwords. + + This command restarts PostgreSQL to pick up new secrets, then updates + database role passwords to match new values. + + Use after rotating secrets to sync the new passwords to the database. + + Examples: + uv run api-forge-cli k8s db sync + """ + console.print_header("Synchronizing PostgreSQL Passwords (Kubernetes)") + success = run_sync(_get_runtime()) + + if not success: + raise typer.Exit(1) + + +@k8s_db_app.command() +@with_error_handling +def backup( + output_dir: Annotated[ + Path | None, + typer.Option( + "--output-dir", + "-o", + help="Local directory for backup files", + ), + ] = None, +) -> None: + """Create a PostgreSQL database backup from Kubernetes. + + Creates a backup by running pg_dump in the pod and copying + the result locally. + + Examples: + uv run api-forge-cli k8s db backup + uv run api-forge-cli k8s db backup --output-dir ./backups + """ + console.print_header("Creating PostgreSQL Backup (Kubernetes)") + + backup_dir = output_dir or Path("./data/postgres-backups") + success, result = run_backup( + _get_runtime(), + output_dir=backup_dir, + superuser_mode=True, + ) + + if not success: + console.error(f"Backup failed: {result}") + raise typer.Exit(1) + + console.print(f"\n[bold green]🎉 Backup created: {result}[/bold green]") + + +@k8s_db_app.command() +@with_error_handling +def reset( + include_temporal: Annotated[ + bool, + typer.Option( + "--temporal/--no-temporal", + help="Also drop Temporal databases and roles", + ), + ] = True, + yes: Annotated[ + bool, + typer.Option( + "--yes", + "-y", + help="Skip confirmation prompt", + ), + ] = False, +) -> None: + """Reset the PostgreSQL database to clean state (DESTRUCTIVE). + + This command drops all application-created databases, roles, and schemas, + returning PostgreSQL to a clean state ready for re-initialization. + + Drops: + - Application database (appdb) + - Application roles (appuser, appowner, backupuser) + - Temporal databases and roles (if --temporal, default) + + Does NOT affect: + - Kubernetes pods or services (use 'k8s down' to clean up) + - PersistentVolumeClaims (use 'k8s down --pvc' to remove) + - System databases (postgres, template0, template1) + + WARNING: This will permanently delete all database data! + + Examples: + uv run api-forge-cli k8s db reset + uv run api-forge-cli k8s db reset --no-temporal # Keep Temporal data + uv run api-forge-cli k8s db reset -y # Skip confirmation + """ + console.print_header("Resetting PostgreSQL Database (Kubernetes)") + + include_temporal = is_temporal_enabled() and include_temporal + + if not yes: + if not console.confirm_action( + "Reset PostgreSQL database", + "This will permanently delete all database data including:\n" + " • All application databases\n" + " • All application roles\n" + " • All tables and data\n" + + (" • Temporal databases and roles\n" if include_temporal else ""), + ): + console.print("[dim]Operation cancelled[/dim]") + raise typer.Exit(0) + + success = run_reset( + _get_runtime(), + include_temporal=include_temporal, + superuser_mode=True, + ) + + if not success: + raise typer.Exit(1) + + console.print("\n[bold green]🎉 PostgreSQL database reset complete![/bold green]") + console.print("\n[dim]To re-initialize:[/dim]") + console.print(" Run 'uv run api-forge-cli k8s db init'") + + +@k8s_db_app.command() +@with_error_handling +def status() -> None: + """Show PostgreSQL health and performance metrics. + + Displays runtime metrics including: + - Connection latency and active connections + - Database sizes and row counts + - Cache hit ratios + - Database uptime + + Works with both bundled Kubernetes PostgreSQL and external databases. + + Examples: + uv run api-forge-cli k8s db status + """ + console.print_header("PostgreSQL Health & Performance") + run_status(_get_runtime(), superuser_mode=True) + + +@k8s_db_app.command() +@with_error_handling +def migrate( + action: Annotated[ + str, + typer.Argument( + help=( + "Migration action: upgrade, downgrade, current, history, revision, " + "heads, merge, show, stamp" + ) + ), + ], + revision: Annotated[ + str | None, + typer.Argument( + help="Target revision (for downgrade) or message (for revision)" + ), + ] = None, + message: Annotated[ + str | None, + typer.Option( + "--message", + "-m", + help=( + "Optional message (used by merge). If omitted, merge will use the second " + "argument as the message when provided, otherwise a default." + ), + ), + ] = None, + merge_revisions: Annotated[ + list[str] | None, + typer.Option( + "--merge-revision", + "-r", + help=( + "Revision(s) to merge (for merge). Can be provided multiple times. " + "If omitted, merges all current heads." + ), + ), + ] = None, + purge: Annotated[ + bool, + typer.Option( + "--purge", + help=( + "For stamp only: purge the version table before stamping. " + "Use with extreme care." + ), + ), + ] = False, + autogenerate: Annotated[ + bool, + typer.Option( + "--autogenerate/--no-autogenerate", + help="Autogenerate migration from model changes (for revision)", + ), + ] = True, + sql: Annotated[ + bool, + typer.Option( + "--sql", + help="Generate SQL output instead of running migration", + ), + ] = False, +) -> None: + """Manage database schema migrations with Alembic. + + Actions: + upgrade [revision] - Apply migrations up to revision (default: head) + downgrade - Rollback to a specific revision + current - Show current migration revision + history - Show migration history + revision - Create a new migration (with --autogenerate) + heads - Show current head revision(s) + merge - Create a merge migration (default: merge all heads) + show - Show a specific migration's details + stamp - Set DB revision without running migrations + + Examples: + # Apply all pending migrations + uv run api-forge-cli k8s db migrate upgrade + + # Apply migrations up to a specific revision + uv run api-forge-cli k8s db migrate upgrade abc123 + + # Rollback to a specific revision + uv run api-forge-cli k8s db migrate downgrade abc123 + + # Rollback one migration + uv run api-forge-cli k8s db migrate downgrade -1 + + # Show current migration state + uv run api-forge-cli k8s db migrate current + + # Show migration history + uv run api-forge-cli k8s db migrate history + + # Create a new migration with autogeneration + uv run api-forge-cli k8s db migrate revision "add user table" + + # Create empty migration template + uv run api-forge-cli k8s db migrate revision "custom migration" --no-autogenerate + + # Generate SQL for upgrade without running it + uv run api-forge-cli k8s db migrate upgrade --sql + + # Show current heads (useful when multiple heads exist) + uv run api-forge-cli k8s db migrate heads + + # Merge all current heads + uv run api-forge-cli k8s db migrate merge --message "merge heads" + + # Merge specific revisions + uv run api-forge-cli k8s db migrate merge --message "merge" \ + -r abc123 -r def456 + + # Show a specific revision + uv run api-forge-cli k8s db migrate show 19becf30b774 + + # Stamp the DB to a revision (no migration execution) + uv run api-forge-cli k8s db migrate stamp head + """ + merge_revisions_normalized: list[str] = merge_revisions or [] + + run_migrate( + _get_runtime(), + action=action, + revision=revision, + message=message, + merge_revisions=merge_revisions_normalized, + purge=purge, + autogenerate=autogenerate, + sql=sql, + ) + + +if __name__ == "__main__": + verify() # For quick testing diff --git a/src/cli/commands/prod.py b/src/cli/commands/prod.py index 0e0e2a5..402e5e7 100644 --- a/src/cli/commands/prod.py +++ b/src/cli/commands/prod.py @@ -4,18 +4,17 @@ environment: starting services, stopping them, and checking status. """ +import subprocess from typing import TYPE_CHECKING, Annotated import typer -from .shared import ( - confirm_action, - console, - get_project_root, - handle_error, - print_header, - with_error_handling, -) +from src.cli.shared.compose import ComposeRunner +from src.cli.shared.console import console, with_error_handling +from src.infra.postgres.connection import get_settings +from src.utils.paths import get_project_root + +from .prod_db import prod_db_app if TYPE_CHECKING: from src.cli.deployment.prod_deployer import ProdDeployer @@ -37,6 +36,49 @@ def _get_deployer() -> "ProdDeployer": return ProdDeployer(console, get_project_root()) +def _get_compose_runner() -> ComposeRunner: + """Create a ComposeRunner for the prod compose file.""" + project_root = get_project_root() + return ComposeRunner( + project_root, + compose_file=project_root / "docker-compose.prod.yml", + project_name="api-forge-prod", + ) + + +def _verify_database_accessible() -> bool: + """Verify PostgreSQL database is accessible. + + Attempts to connect to the database and run a simple query. + Returns True if successful, False otherwise. + """ + try: + from src.infra.docker_compose.postgres_connection import ( + get_docker_compose_postgres_connection, + ) + + settings = get_settings() + conn = get_docker_compose_postgres_connection(settings) + + # Test connection with a short timeout + success, msg = conn.test_connection() + + if success: + console.ok(f"Database accessible: {msg[:60]}...") + else: + console.error(f"Database check failed: {msg}") + + return success + + except ImportError: + # psycopg not installed, skip check + console.warn("Database check skipped (psycopg not installed)") + return True + except Exception as e: + console.error(f"Database check failed: {e}") + return False + + # --------------------------------------------------------------------------- # Typer App # --------------------------------------------------------------------------- @@ -77,10 +119,18 @@ def up( help="Force recreate containers (useful for secret rotation)", ), ] = False, + skip_db_check: Annotated[ + bool, + typer.Option( + "--skip-db-check", + help="Skip PostgreSQL verification before deployment", + ), + ] = False, ) -> None: """Start the production Docker Compose environment. This command: + - Verifies PostgreSQL is accessible (unless --skip-db-check) - Ensures required data directories exist - Validates and cleans up stale bind-mount volumes - Builds the application Docker image (unless --skip-build) @@ -91,8 +141,31 @@ def up( uv run api-forge-cli prod up uv run api-forge-cli prod up --skip-build --no-wait uv run api-forge-cli prod up --force-recreate # For secret rotation + uv run api-forge-cli prod up --skip-db-check # Skip database verification """ - print_header("Starting Production Environment") + from src.infra.utils.service_config import is_bundled_postgres_enabled + + console.print_header("Starting Production Environment") + + # Verify database is accessible before deploying + if not skip_db_check: + if not _verify_database_accessible(): + console.error("Database verification failed.") + console.print( + "[dim]Please ensure PostgreSQL is running and accessible.[/dim]\n" + ) + if is_bundled_postgres_enabled(): + console.print( + "[dim]For bundled PostgreSQL, run:[/dim]\n" + " uv run api-forge-cli db create\n" + " uv run api-forge-cli db init\n" + ) + else: + console.print( + "[dim]For external PostgreSQL, verify DATABASE_URL in .env[/dim]\n" + ) + console.print("[dim]Use --skip-db-check to bypass this verification.[/dim]") + raise typer.Exit(1) deployer = _get_deployer() deployer.deploy( @@ -132,10 +205,10 @@ def down( uv run api-forge-cli prod down --volumes # Remove data too uv run api-forge-cli prod down -v -y # Remove data without prompt """ - print_header("Stopping Production Environment") + console.print_header("Stopping Production Environment") if volumes and not yes: - if not confirm_action( + if not console.confirm_action( "Remove data volumes", "This will permanently delete all production data including:\n" " • PostgreSQL database\n" @@ -160,7 +233,7 @@ def status() -> None: Examples: uv run api-forge-cli prod status """ - print_header("Production Environment Status") + console.print_header("Production Environment Status") deployer = _get_deployer() deployer.show_status() @@ -203,40 +276,23 @@ def logs( uv run api-forge-cli prod logs app -f # Follow app logs uv run api-forge-cli prod logs -n 50 # Last 50 lines """ - import subprocess - project_root = get_project_root() compose_file = project_root / "docker-compose.prod.yml" if not compose_file.exists(): - handle_error(f"Compose file not found: {compose_file}") + console.handle_error(f"Compose file not found: {compose_file}") raise typer.Exit(1) - cmd = [ - "docker", - "compose", - "-p", - "api-forge-prod", - "-f", - str(compose_file), - "logs", - f"--tail={tail}", - ] - - if follow: - cmd.append("--follow") - if service: - cmd.append(service) console.print(f"[dim]Showing logs for service: {service}[/dim]\n") else: console.print("[dim]Showing logs for all production services[/dim]\n") try: - subprocess.run(cmd, check=True) + runner = _get_compose_runner() + runner.logs(service=service, follow=follow, tail=tail) except subprocess.CalledProcessError as e: - handle_error(f"Failed to retrieve logs: {e}") - raise typer.Exit(1) from e + console.handle_error(f"Failed to retrieve logs: {e}") except KeyboardInterrupt: console.print("\n[dim]Log streaming stopped[/dim]") @@ -268,59 +324,29 @@ def restart( uv run api-forge-cli prod restart app # Just restart app uv run api-forge-cli prod restart --force-recreate """ - import subprocess - project_root = get_project_root() compose_file = project_root / "docker-compose.prod.yml" if not compose_file.exists(): - handle_error(f"Compose file not found: {compose_file}") - raise typer.Exit(1) + console.handle_error(f"Compose file not found: {compose_file}") if service: - console.print(f"[cyan]Restarting service: {service}[/cyan]") - cmd = [ - "docker", - "compose", - "-p", - "api-forge-prod", - "-f", - str(compose_file), - "restart", - service, - ] + console.info(f"Restarting service: {service}") elif force_recreate: # Full restart with force-recreate - console.print("[cyan]Force restarting all production services...[/cyan]") - cmd = [ - "docker", - "compose", - "-p", - "api-forge-prod", - "-f", - str(compose_file), - "up", - "-d", - "--force-recreate", - ] + console.info("Force restarting all production services...") else: - console.print("[cyan]Restarting all production services...[/cyan]") - cmd = [ - "docker", - "compose", - "-p", - "api-forge-prod", - "-f", - str(compose_file), - "restart", - ] + console.info("Restarting all production services...") try: - subprocess.run(cmd, check=True) - console.print("[green]✓[/green] Restart complete") + runner = _get_compose_runner() + if force_recreate and not service: + runner.run(["up", "-d", "--force-recreate"], check=True) + else: + runner.restart(service=service) + console.ok("Restart complete") except subprocess.CalledProcessError as e: - handle_error(f"Failed to restart services: {e}") - raise typer.Exit(1) from e + console.handle_error(f"Failed to restart services: {e}") @prod_app.command() @@ -350,37 +376,27 @@ def build( uv run api-forge-cli prod build app # Just build app uv run api-forge-cli prod build --no-cache """ - import subprocess - project_root = get_project_root() compose_file = project_root / "docker-compose.prod.yml" if not compose_file.exists(): - handle_error(f"Compose file not found: {compose_file}") - raise typer.Exit(1) - - cmd = [ - "docker", - "compose", - "-p", - "api-forge-prod", - "-f", - str(compose_file), - "build", - ] - - if no_cache: - cmd.append("--no-cache") + console.handle_error(f"Compose file not found: {compose_file}") if service: - cmd.append(service) - console.print(f"[cyan]Building service: {service}[/cyan]") + console.info(f"Building service: {service}") else: - console.print("[cyan]Building all production images...[/cyan]") + console.info("Building all production images...") try: - subprocess.run(cmd, check=True) - console.print("[green]✓[/green] Build complete") + runner = _get_compose_runner() + runner.build(service=service, no_cache=no_cache) + console.ok("Build complete") except subprocess.CalledProcessError as e: - handle_error(f"Build failed: {e}") - raise typer.Exit(1) from e + console.handle_error(f"Build failed: {e}") + + +# --------------------------------------------------------------------------- +# Register Subcommands +# --------------------------------------------------------------------------- + +prod_app.add_typer(prod_db_app, name="db") diff --git a/src/cli/commands/prod_db.py b/src/cli/commands/prod_db.py new file mode 100644 index 0000000..201185e --- /dev/null +++ b/src/cli/commands/prod_db.py @@ -0,0 +1,758 @@ +"""PostgreSQL database management for Docker Compose deployments. + +This module provides db subcommands under 'prod' for managing PostgreSQL +databases in Docker Compose environments. +""" + +import subprocess +import time +from pathlib import Path +from typing import Annotated + +import typer + +from src.cli.commands.db import ( + DbRuntime, + get_compose_runtime, + run_backup, + run_init, + run_migrate, + run_reset, + run_status, + run_sync, + run_verify, +) +from src.cli.commands.db_utils import ( + configure_external_database, + read_env_example_values, + update_bundled_postgres_config, + update_env_file, +) +from src.cli.deployment.status_display import is_temporal_enabled +from src.cli.shared.console import console, with_error_handling +from src.utils.paths import get_project_root + + +def _get_compose_file() -> Path: + """Get the docker-compose.prod.yml file path.""" + return get_project_root() / "docker-compose.prod.yml" + + +def _get_runtime() -> DbRuntime: + """Return the Docker Compose DB runtime.""" + return get_compose_runtime() + + +def _run_compose_command( + args: list[str], capture: bool = False +) -> subprocess.CompletedProcess[str]: + """Run a docker-compose command with the prod compose file.""" + compose_file = _get_compose_file() + # Use the same fixed project name as ProdDeployer to avoid cross-command + # network/volume label conflicts (e.g., application_internal). + cmd = ["docker", "compose", "-p", "api-forge-prod", "-f", str(compose_file)] + args + if capture: + return subprocess.run( + cmd, capture_output=True, text=True, cwd=get_project_root() + ) + return subprocess.run(cmd, cwd=get_project_root(), capture_output=True, text=True) + + +# --------------------------------------------------------------------------- +# Typer App +# --------------------------------------------------------------------------- + +prod_db_app = typer.Typer( + name="db", + help="PostgreSQL database management for Docker Compose.", + no_args_is_help=True, +) + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + + +@prod_db_app.command() +@with_error_handling +def create( + # Mode selection (mutually exclusive) + bundled: Annotated[ + bool, + typer.Option( + "--bundled", + help="Deploy bundled PostgreSQL via Docker Compose", + ), + ] = False, + external: Annotated[ + bool, + typer.Option( + "--external", + help="Configure connection to external PostgreSQL", + ), + ] = False, + # External mode parameters + connection_string: Annotated[ + str | None, + typer.Option( + "--connection-string", + "-c", + help="Full PostgreSQL connection string (postgres://user:pass@host:port/db)", + ), + ] = None, + host: Annotated[ + str | None, + typer.Option( + "--host", + "-H", + help="Database host (overrides connection string)", + ), + ] = None, + port: Annotated[ + str | None, + typer.Option( + "--port", + "-P", + help="Database port (overrides connection string)", + ), + ] = None, + username: Annotated[ + str | None, + typer.Option( + "--username", + "-u", + help="Database username (overrides connection string)", + ), + ] = None, + password: Annotated[ + str | None, + typer.Option( + "--password", + "-p", + help="Database password (overrides connection string)", + ), + ] = None, + database: Annotated[ + str | None, + typer.Option( + "--database", + "-d", + help="Database name (overrides connection string)", + ), + ] = None, + sslmode: Annotated[ + str | None, + typer.Option( + "--sslmode", + help="SSL mode (e.g., require, verify-full)", + ), + ] = None, + tls_ca: Annotated[ + str | None, + typer.Option( + "--tls-ca", + help="Path to TLS CA certificate file for external PostgreSQL (e.g., Aiven CA)", + ), + ] = None, + # Bundled mode parameters + skip_build: Annotated[ + bool, + typer.Option( + "--skip-build", + help="Skip building the PostgreSQL image (bundled mode only)", + ), + ] = False, + wait: Annotated[ + bool, + typer.Option( + "--wait/--no-wait", + help="Wait for PostgreSQL to be healthy (bundled mode only)", + ), + ] = True, +) -> None: + """Configure PostgreSQL database for Docker Compose deployment. + + Two modes are available: + + --bundled: Deploy a bundled PostgreSQL instance via Docker Compose. + This builds the PostgreSQL image and starts the container with security settings. + + --external: Configure connection to an external PostgreSQL instance (e.g., Aiven, RDS). + This updates .env and config.yaml with connection details. + + Examples: + # Deploy bundled PostgreSQL + uv run api-forge-cli prod db create --bundled + uv run api-forge-cli prod db create --bundled --skip-build + + # Configure external PostgreSQL with connection string + uv run api-forge-cli prod db create --external \\ + --connection-string "postgres://admin:secret@db.example.com:5432/mydb?sslmode=require" + + # Configure external PostgreSQL with individual parameters + uv run api-forge-cli prod db create --external \\ + --host db.example.com --port 5432 \\ + --username admin --password secret --database mydb + + # Mix: connection string with override + uv run api-forge-cli prod db create --external \\ + --connection-string "postgres://user:pass@host:5432/db" \\ + --password new-secret # Override password from connection string + """ + # Validate mode selection + if bundled and external: + console.print("[red]❌ Cannot use both --bundled and --external[/red]") + raise typer.Exit(1) + + if not bundled and not external: + console.print("[red]❌ Must specify either --bundled or --external[/red]") + console.print( + "[dim]Use --bundled to deploy PostgreSQL via Docker Compose[/dim]" + ) + console.print("[dim]Use --external to configure an external database[/dim]") + raise typer.Exit(1) + + if external: + configure_external_database( + connection_string=connection_string, + host=host, + port=port, + username=username, + password=password, + database=database, + sslmode=sslmode, + tls_ca=tls_ca, + next_steps_cmd_prefix="prod", + ) + else: + _create_bundled(skip_build=skip_build, wait=wait) + + +def _create_bundled(*, skip_build: bool, wait: bool) -> None: + """Create and start the bundled PostgreSQL container.""" + console.print_header("Creating PostgreSQL Container (Docker Compose)") + + # If a previous run created the shared network under a different compose project + # name, Docker Compose will error with a label mismatch. + # We proactively remove that stale network so this command can proceed. + network_name = "application_internal" + expected_project = "api-forge-prod" + inspect = subprocess.run( + [ + "docker", + "network", + "inspect", + network_name, + "--format", + '{{ index .Labels "com.docker.compose.project" }}', + ], + capture_output=True, + text=True, + check=False, + ) + if inspect.returncode == 0: + existing_project = (inspect.stdout or "").strip() + if existing_project and existing_project != expected_project: + console.print( + f"[yellow]⚠ Removing stale network '{network_name}' from compose project '{existing_project}'[/yellow]" + ) + subprocess.run( + ["docker", "network", "rm", network_name], + capture_output=True, + text=True, + check=False, + ) + + compose_file = _get_compose_file() + if not compose_file.exists(): + console.error("docker-compose.prod.yml not found") + raise typer.Exit(1) + + # Ensure data directories exist before starting container + from src.cli.deployment.prod_deployer import ProdDeployer + + deployer = ProdDeployer(console, get_project_root()) + deployer._ensure_required_directories() + + # Read bundled defaults from .env.example + bundled_keys = ["PRODUCTION_DATABASE_URL", "PG_SUPERUSER", "PG_DB"] + bundled_defaults = read_env_example_values(bundled_keys) + + if not bundled_defaults: + console.print("[red]❌ Could not read bundled defaults from .env.example[/red]") + raise typer.Exit(1) + + # Update .env with bundled defaults from .env.example + console.info("Updating .env file for bundled PostgreSQL...") + update_env_file(bundled_defaults) + console.ok(".env file updated") + + # Update config.yaml + console.info("Updating config.yaml...") + update_bundled_postgres_config(enabled=True) + console.ok("config.yaml updated (bundled_postgres.enabled=true)") + + # Check if container already exists (exact name match) + check_result = subprocess.run( + [ + "docker", + "ps", + "-a", + "--filter", + "name=^api-forge-postgres$", + "--format", + "{{.Names}}", + ], + capture_output=True, + text=True, + ) + container_exists = check_result.stdout.strip() == "api-forge-postgres" + + if container_exists: + console.info( + "Container 'api-forge-postgres' already exists, checking status..." + ) + status_result = subprocess.run( + [ + "docker", + "inspect", + "--format", + "{{.State.Status}}", + "api-forge-postgres", + ], + capture_output=True, + text=True, + ) + + if status_result.returncode != 0: + console.error( + f"Failed to inspect container (it may have been removed):\n{status_result.stderr}" + ) + console.info("Recreating container...") + # Fall through to creation logic below + container_exists = False + else: + status = status_result.stdout.strip() + + if status == "running": + console.print("[yellow]⚠️ Container is already running[/yellow]") + console.print( + "[dim]Use 'uv run api-forge-cli prod db status' to check health[/dim]" + ) + return + else: + console.info(f"Container status: {status}, starting it...") + # Use docker start for existing containers instead of compose up + result = subprocess.run( + ["docker", "start", "api-forge-postgres"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + console.error( + f"Failed to start PostgreSQL container:\n{result.stderr}" + ) + raise typer.Exit(1) + + if not container_exists: + # Build image if needed + if not skip_build: + console.info("Building PostgreSQL image...") + result = _run_compose_command(["build", "postgres"]) + if result.returncode != 0: + console.error(f"Failed to build PostgreSQL image:\n{result.stderr}") + raise typer.Exit(1) + console.ok("PostgreSQL image built") + + # Start postgres with profile + console.info("Starting PostgreSQL container...") + result = _run_compose_command(["--profile", "postgres", "up", "-d", "postgres"]) + if result.returncode != 0: + console.error(f"Failed to start PostgreSQL container:\n{result.stderr}") + raise typer.Exit(1) + + # Wait for health check + if wait: + console.info("Waiting for PostgreSQL to be healthy...") + max_wait = 60 + start = time.time() + + while time.time() - start < max_wait: + result = subprocess.run( + [ + "docker", + "inspect", + "--format", + "{{.State.Health.Status}}", + "api-forge-postgres", + ], + capture_output=True, + text=True, + ) + health = result.stdout.strip() + if health == "healthy": + console.ok("PostgreSQL is healthy") + break + elif health == "unhealthy": + console.error("PostgreSQL is unhealthy") + raise typer.Exit(1) + time.sleep(2) + else: + console.error("Timed out waiting for PostgreSQL") + raise typer.Exit(1) + + console.print("\n[bold green]🎉 PostgreSQL container created![/bold green]") + + # Auto-initialize database after successful deployment + console.info("\nInitializing database...") + + try: + success = run_init(_get_runtime()) + if not success: + console.error("Database initialization failed") + console.print("\n[dim]You can retry with:[/dim]") + console.print(" uv run api-forge-cli prod db init") + raise typer.Exit(1) + + console.ok("Database initialized successfully") + + except Exception as e: + console.error(f"Database initialization failed: {e}") + console.print("\n[dim]You can retry with:[/dim]") + console.print(" uv run api-forge-cli prod db init") + raise typer.Exit(1) from e + console.print("\n[dim]Next steps:[/dim]") + console.print(" - Run 'uv run api-forge-cli prod db verify' to verify setup") + + +@prod_db_app.command(name="init") +@with_error_handling +def init_db() -> None: + """Initialize the PostgreSQL database with roles and schema. + + This command: + - Creates the owner, app user, and read-only roles + - Creates the application database + - Sets up the schema with proper privileges + + Credentials are prompted at runtime (never stored in files). + + Examples: + uv run api-forge-cli prod db init + uv run api-forge-cli prod db init --temporal + """ + console.print_header("Initializing PostgreSQL Database (Docker Compose)") + success = run_init(_get_runtime()) + + if not success: + raise typer.Exit(1) + + +@prod_db_app.command() +@with_error_handling +def verify() -> None: + """Verify PostgreSQL database setup and configuration. + + This command checks: + - Role existence and attributes + - Database and schema ownership + - Table and sequence privileges + - TLS configuration + - Temporal roles (if enabled) + + Examples: + uv run api-forge-cli prod db verify + """ + console.print_header("Verifying PostgreSQL Configuration (Docker Compose)") + success = run_verify(_get_runtime(), superuser_mode=False) + + if not success: + console.info( + 'Please run "uv run api-forge-cli prod db init" to re-initialize the database.' + ) + raise typer.Exit(1) + + +@prod_db_app.command() +@with_error_handling +def sync() -> None: + """Synchronize PostgreSQL role passwords. + + This command updates database role passwords to match new values. + Use after rotating secrets to sync the new passwords to the database. + + Credentials are prompted at runtime. + + Examples: + uv run api-forge-cli prod db sync + uv run api-forge-cli prod db sync --temporal + """ + console.print_header("Synchronizing PostgreSQL Passwords (Docker Compose)") + success = run_sync(_get_runtime()) + + if not success: + raise typer.Exit(1) + + +@prod_db_app.command() +@with_error_handling +def backup( + output_dir: Annotated[ + Path | None, + typer.Option( + "--output-dir", + "-o", + help="Directory for backup files", + ), + ] = None, +) -> None: + """Create a PostgreSQL database backup. + + Creates both custom format (.dump) and compressed SQL (.sql.gz) backups + with SHA256 checksums. Uses the read-only user for backups. + + Examples: + uv run api-forge-cli prod db backup + uv run api-forge-cli prod db backup --output-dir ./backups + """ + console.print_header("Creating PostgreSQL Backup (Docker Compose)") + backup_dir = output_dir or Path("./data/postgres-backups") + success, result = run_backup( + _get_runtime(), + output_dir=backup_dir, + superuser_mode=False, + ) + + if not success: + console.error(f"Backup failed: {result}") + raise typer.Exit(1) + + console.print(f"\n[bold green]🎉 Backup created: {result}[/bold green]") + + +@prod_db_app.command() +@with_error_handling +def reset( + include_temporal: Annotated[ + bool, + typer.Option( + "--temporal/--no-temporal", + help="Also drop Temporal databases and roles", + ), + ] = True, + yes: Annotated[ + bool, + typer.Option( + "--yes", + "-y", + help="Skip confirmation prompt", + ), + ] = False, +) -> None: + """Reset the PostgreSQL database to clean state (DESTRUCTIVE). + + This command drops all application-created databases, roles, and schemas, + returning PostgreSQL to a clean state ready for re-initialization. + + Drops: + - Application database (appdb) + - Application roles (appuser, appowner, backupuser) + - Temporal databases and roles (if --temporal, default) + + Does NOT affect: + - Docker containers (use 'prod down' to stop containers) + - Docker volumes (use 'prod down --volumes' to remove volumes) + - System databases (postgres, template0, template1) + + WARNING: This will permanently delete all database data! + + Examples: + uv run api-forge-cli prod db reset + uv run api-forge-cli prod db reset --no-temporal # Keep Temporal data + uv run api-forge-cli prod db reset -y # Skip confirmation + """ + console.print_header("Resetting PostgreSQL Database (Docker Compose)") + + include_temporal = is_temporal_enabled() and include_temporal + + if not yes: + if not console.confirm_action( + "Reset PostgreSQL database", + "This will permanently delete all database data including:\n" + " • All application databases\n" + " • All application roles\n" + " • All tables and data\n" + + (" • Temporal databases and roles\n" if include_temporal else ""), + ): + console.print("[dim]Operation cancelled[/dim]") + raise typer.Exit(0) + + success = run_reset( + _get_runtime(), + include_temporal=include_temporal, + superuser_mode=False, + ) + + if not success: + raise typer.Exit(1) + + console.print("\n[bold green]🎉 PostgreSQL database reset complete![/bold green]") + console.print("\n[dim]To re-initialize:[/dim]") + console.print(" Run 'uv run api-forge-cli prod db init'") + + +@prod_db_app.command() +@with_error_handling +def status() -> None: + """Show PostgreSQL health and performance metrics. + + Displays runtime metrics including: + - Connection latency and active connections + - Database sizes and row counts + - Cache hit ratios + - Database uptime + + Works with both bundled Docker Compose PostgreSQL and external databases. + + Examples: + uv run api-forge-cli prod db status + """ + console.print_header("PostgreSQL Health & Performance") + run_status(_get_runtime(), superuser_mode=False) + + +@prod_db_app.command() +@with_error_handling +def migrate( + action: Annotated[ + str, + typer.Argument( + help=( + "Migration action: upgrade, downgrade, current, history, revision, " + "heads, merge, show, stamp" + ) + ), + ], + revision: Annotated[ + str | None, + typer.Argument( + help="Target revision (for downgrade) or message (for revision)" + ), + ] = None, + message: Annotated[ + str | None, + typer.Option( + "--message", + "-m", + help=( + "Optional message (used by merge). If omitted, merge will use the second " + "argument as the message when provided, otherwise a default." + ), + ), + ] = None, + merge_revisions: Annotated[ + list[str] | None, + typer.Option( + "--merge-revision", + "-r", + help=( + "Revision(s) to merge (for merge). Can be provided multiple times. " + "If omitted, merges all current heads." + ), + ), + ] = None, + purge: Annotated[ + bool, + typer.Option( + "--purge", + help=( + "For stamp only: purge the version table before stamping. " + "Use with extreme care." + ), + ), + ] = False, + autogenerate: Annotated[ + bool, + typer.Option( + "--autogenerate/--no-autogenerate", + help="Autogenerate migration from model changes (for revision)", + ), + ] = True, + sql: Annotated[ + bool, + typer.Option( + "--sql", + help="Generate SQL output instead of running migration", + ), + ] = False, +) -> None: + """Manage database schema migrations with Alembic. + + Actions: + upgrade [revision] - Apply migrations up to revision (default: head) + downgrade - Rollback to a specific revision + current - Show current migration revision + history - Show migration history + revision - Create a new migration (with --autogenerate) + heads - Show current head revision(s) + merge - Create a merge migration (default: merge all heads) + show - Show a specific migration's details + stamp - Set DB revision without running migrations + + Examples: + # Apply all pending migrations + uv run api-forge-cli prod db migrate upgrade + + # Apply migrations up to a specific revision + uv run api-forge-cli prod db migrate upgrade abc123 + + # Rollback to a specific revision + uv run api-forge-cli prod db migrate downgrade abc123 + + # Rollback one migration + uv run api-forge-cli prod db migrate downgrade -1 + + # Show current migration state + uv run api-forge-cli prod db migrate current + + # Show migration history + uv run api-forge-cli prod db migrate history + + # Create a new migration with autogeneration + uv run api-forge-cli prod db migrate revision "add user table" + + # Create empty migration template + uv run api-forge-cli prod db migrate revision "custom migration" --no-autogenerate + + # Generate SQL for upgrade without running it + uv run api-forge-cli prod db migrate upgrade --sql + + # Show current heads (useful when multiple heads exist) + uv run api-forge-cli prod db migrate heads + + # Merge all current heads + uv run api-forge-cli prod db migrate merge --message "merge heads" + + # Merge specific revisions + uv run api-forge-cli prod db migrate merge --message "merge" \ + -r abc123 -r def456 + + # Show a specific revision + uv run api-forge-cli prod db migrate show 19becf30b774 + + # Stamp the DB to a revision (no migration execution) + uv run api-forge-cli prod db migrate stamp head + """ + merge_revisions_normalized: list[str] = merge_revisions or [] + + run_migrate( + _get_runtime(), + action=action, + revision=revision, + message=message, + merge_revisions=merge_revisions_normalized, + purge=purge, + autogenerate=autogenerate, + sql=sql, + ) diff --git a/src/cli/commands/secrets.py b/src/cli/commands/secrets.py index a4b8849..5504c70 100644 --- a/src/cli/commands/secrets.py +++ b/src/cli/commands/secrets.py @@ -7,7 +7,8 @@ from rich.panel import Panel from rich.table import Table -from .shared import confirm_destructive_action, console, get_project_root +from src.cli.shared.console import console +from src.utils.paths import get_project_root # Create the secrets command group secrets_app = typer.Typer(help="🔐 Secrets management commands") @@ -79,10 +80,8 @@ def generate( # Check if script exists if not generate_script.exists(): - console.print( - "[red]❌ Error: generate_secrets.sh not found at:[/red]", - f" {generate_script}", - ) + console.error("generate_secrets.sh not found at:") + console.print(f" {generate_script}") raise typer.Exit(1) # Confirm if using --force (destructive overwrite) @@ -99,7 +98,7 @@ def generate( "\n • TLS certificates (Root CA, Intermediate CA, service certs)" ) - if not confirm_destructive_action( + if not console.confirm_action( action="Regenerate ALL secrets", details=details, extra_warning="⚠️ Existing secrets will be permanently overwritten!", @@ -110,9 +109,11 @@ def generate( # Run generate_secrets.sh if pki: - console.print("\n[cyan]🔐 Generating secrets and PKI certificates...[/cyan]\n") + console.info("🔐 Generating secrets and PKI certificates...") + console.print() else: - console.print("\n[cyan]🔐 Generating secrets...[/cyan]\n") + console.info("🔐 Generating secrets...") + console.print() try: # Make script executable @@ -142,8 +143,8 @@ def generate( ) if result.returncode != 0: - console.print( - f"\n[red]❌ Secret generation failed with exit code {result.returncode}[/red]" + console.error( + f"Secret generation failed with exit code {result.returncode}" ) raise typer.Exit(1) @@ -165,10 +166,10 @@ def generate( ) except subprocess.CalledProcessError as e: - console.print(f"[red]❌ Error running generate_secrets.sh: {e}[/red]") + console.error(f"Error running generate_secrets.sh: {e}") raise typer.Exit(1) from e except Exception as e: - console.print(f"[red]❌ Unexpected error: {e}[/red]") + console.error(f"Unexpected error: {e}") raise typer.Exit(1) from e @@ -275,17 +276,13 @@ def list( certs_exist = certs_dir.exists() and any(certs_dir.iterdir()) if keys_exist and certs_exist: - console.print("\n[green]✅ All secrets appear to be generated.[/green]") + console.ok("All secrets appear to be generated.") elif not keys_exist and not certs_exist: - console.print( - "\n[yellow]⚠️ No secrets found. Run:[/yellow]", - " uv run api-forge-cli secrets generate", - ) + console.warn("No secrets found. Run:") + console.print(" uv run api-forge-cli secrets generate") else: - console.print( - "\n[yellow]⚠️ Some secrets are missing. Run:[/yellow]", - " uv run api-forge-cli secrets generate", - ) + console.warn("Some secrets are missing. Run:") + console.print(" uv run api-forge-cli secrets generate") @secrets_app.command() @@ -378,25 +375,23 @@ def verify() -> None: if missing_keys: has_errors = True - console.print("\n[red]❌ Missing key files:[/red]") + console.error("Missing key files:") for filename in missing_keys: console.print(f" • {filename}") if missing_certs: has_errors = True - console.print("\n[red]❌ Missing certificate files:[/red]") + console.error("Missing certificate files:") for filename in missing_certs: console.print(f" • {filename}") if unreadable: has_errors = True - console.print("\n[red]❌ Unreadable files:[/red]") + console.error("Unreadable files:") for filename in unreadable: console.print(f" • {filename}") if has_errors: - console.print( - "\n[yellow]💡 To generate missing secrets, run:[/yellow]", - " uv run api-forge-cli secrets generate", - ) + console.warn("💡 To generate missing secrets, run:") + console.print(" uv run api-forge-cli secrets generate") raise typer.Exit(1) diff --git a/src/cli/commands/shared.py b/src/cli/commands/shared.py deleted file mode 100644 index ae2a862..0000000 --- a/src/cli/commands/shared.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Shared utilities for CLI commands. - -This module provides common utilities used across all command modules, -including console output, confirmation dialogs, and path resolution. -""" - -from collections.abc import Callable -from pathlib import Path - -import typer -from rich.console import Console -from rich.panel import Panel - -# Shared console instance for consistent output -console = Console() - - -def get_project_root() -> Path: - """Get the project root directory. - - Walks up from the module location to find the project root, - identified by the presence of pyproject.toml. - - Returns: - Path to the project root directory - """ - current = Path(__file__).resolve() - - # Walk up the directory tree looking for pyproject.toml - for parent in [current, *current.parents]: - if (parent / "pyproject.toml").exists(): - return parent - - # Fallback to four levels up (src/cli/commands/shared.py -> project root) - return Path(__file__).parent.parent.parent.parent - - -def confirm_action( - action: str, - details: str | None = None, - extra_warning: str | None = None, - force: bool = False, -) -> bool: - """Prompt user to confirm a potentially destructive action. - - Args: - action: Description of the action (e.g., "Stop all services") - details: Additional details about what will be affected - extra_warning: Extra warning message (e.g., for data loss) - force: If True, skip the confirmation prompt - - Returns: - True if the user confirmed, False otherwise - """ - if force: - return True - - # Build warning message - warning_lines = [f"[bold red]⚠️ {action}[/bold red]"] - - if details: - warning_lines.append(f"\n{details}") - - if extra_warning: - warning_lines.append(f"\n[yellow]{extra_warning}[/yellow]") - - console.print( - Panel( - "\n".join(warning_lines), - title="Confirmation Required", - border_style="red", - ) - ) - - try: - response = console.input( - "\n[bold]Are you sure you want to proceed?[/bold] \\[y/N]: " - ) - return response.strip().lower() in ("y", "yes") - except (KeyboardInterrupt, EOFError): - console.print("\n[dim]Cancelled.[/dim]") - return False - - -# Alias for backward compatibility -confirm_destructive_action = confirm_action - - -def handle_error(message: str, details: str | None = None, exit_code: int = 1) -> None: - """Handle an error by printing a message and exiting. - - Args: - message: Error message to display - details: Optional additional details - exit_code: Exit code to use - """ - console.print(f"\n[bold red]❌ {message}[/bold red]\n") - if details: - console.print(Panel(details, title="Details", border_style="red")) - raise typer.Exit(exit_code) - - -def print_header(title: str, style: str = "blue") -> None: - """Print a styled header panel. - - Args: - title: Header title text - style: Border style color - """ - console.print( - Panel.fit( - f"[bold {style}]{title}[/bold {style}]", - border_style=style, - ) - ) - - -def with_error_handling(func: Callable[..., None]) -> Callable[..., None]: - """Decorator to wrap command functions with standard error handling. - - Catches common exceptions and formats them consistently. - - Args: - func: The command function to wrap - - Returns: - Wrapped function with error handling - """ - from functools import wraps - - from src.cli.deployment.helm_deployer.image_builder import DeploymentError - - @wraps(func) - def wrapper(*args: object, **kwargs: object) -> None: - try: - func(*args, **kwargs) - except DeploymentError as e: - handle_error(e.message, e.details) - except KeyboardInterrupt: - console.print("\n[dim]Operation cancelled by user.[/dim]") - raise typer.Exit(130) from None - - return wrapper diff --git a/src/cli/commands/users.py b/src/cli/commands/users.py index a60e39a..142001c 100644 --- a/src/cli/commands/users.py +++ b/src/cli/commands/users.py @@ -4,10 +4,11 @@ from rich.prompt import Confirm from rich.table import Table +from src.cli.shared.console import ( + console, +) from src.dev.keycloak_client import KeycloakClient -from .shared import console - # Create the users subcommand app users_app = typer.Typer(help="Manage Keycloak users in development environment") diff --git a/src/cli/context.py b/src/cli/context.py new file mode 100644 index 0000000..2f31c3f --- /dev/null +++ b/src/cli/context.py @@ -0,0 +1,52 @@ +"""CLI context and dependency container.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import click +import typer + +from src.cli.deployment.shell_commands import ShellCommands +from src.cli.shared.console import CLIConsole, console +from src.infra.constants import DeploymentConstants, DeploymentPaths +from src.infra.k8s import get_k8s_controller_sync +from src.infra.k8s.controller import KubernetesControllerSync +from src.utils.paths import get_project_root + + +@dataclass(frozen=True) +class CLIContext: + """Runtime dependencies for CLI commands.""" + + console: CLIConsole + project_root: Path + commands: ShellCommands + k8s_controller: KubernetesControllerSync + constants: DeploymentConstants + paths: DeploymentPaths + + +def build_cli_context() -> CLIContext: + """Build a fresh CLIContext.""" + project_root = get_project_root() + constants = DeploymentConstants() + paths = DeploymentPaths(project_root) + + return CLIContext( + console=console, + project_root=project_root, + commands=ShellCommands(project_root), + k8s_controller=get_k8s_controller_sync(), + constants=constants, + paths=paths, + ) + + +def get_cli_context(ctx: typer.Context | None = None) -> CLIContext: + """Return the CLIContext from Typer, falling back to a new instance.""" + context = ctx or click.get_current_context(silent=True) + if context and isinstance(context.obj, CLIContext): + return context.obj + return build_cli_context() diff --git a/src/cli/deployment/base.py b/src/cli/deployment/base.py index 39a11b5..b6c8f4a 100644 --- a/src/cli/deployment/base.py +++ b/src/cli/deployment/base.py @@ -8,14 +8,16 @@ from typing import Any from dotenv import load_dotenv -from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn +from src.cli.deployment.health_checks import HealthChecker +from src.cli.shared.console import CLIConsole + class BaseDeployer(ABC): """Abstract base class for all deployers.""" - def __init__(self, console: Console, project_root: Path): + def __init__(self, console: CLIConsole, project_root: Path): """Initialize the deployer. Args: @@ -50,6 +52,80 @@ def show_status(self) -> None: """Display the current status of the deployment.""" pass + @abstractmethod + def deploy_secrets(self) -> bool: + """Deploy secrets for the environment. + + Returns: + True if secrets were deployed successfully, False otherwise + """ + pass + + @abstractmethod + def restart_resource(self, label: str, resource_type: str, timeout: int) -> bool: + """Restart a resource and wait for it to be healthy. + + Args: + label: Resource identifier (container name or k8s resource name) + resource_type: Type of resource ('statefulset' or 'deployment' for k8s, 'container' for docker-compose) + timeout: Maximum time to wait for resource to be healthy (in seconds) + + Returns: + True if restart succeeded and resource is healthy, False otherwise + """ + pass + + def restart_container( + self, label: str, health_checker: HealthChecker, timeout: int = 120 + ) -> bool: + """Restart a Docker container and wait for it to be healthy.""" + self.info(f"Restarting container: {label}...") + + stop_result = self.run_command( + ["docker", "stop", label], + check=False, + capture_output=True, + ) + + if stop_result and stop_result.returncode != 0: + self.console.print(f"[yellow]Warning: Could not stop {label}[/yellow]") + + start_result = self.run_command( + ["docker", "start", label], + check=False, + capture_output=True, + ) + + if not start_result or start_result.returncode != 0: + self.console.print(f"[red]Failed to start {label}[/red]") + return False + + self.console.print(f"[dim]Waiting for {label} to be healthy...[/dim]") + + def check_health() -> bool: + is_healthy, _ = health_checker.check_container_health(label) + return is_healthy + + is_healthy = health_checker.wait_for_condition( + check_health, timeout=timeout, interval=3, service_name=label + ) + + if is_healthy: + self.success(f"Container {label} restarted and healthy") + return True + + result = self.run_command( + ["docker", "inspect", "-f", "{{.State.Running}}", label], + capture_output=True, + check=False, + ) + if result and result.stdout and result.stdout.strip().lower() == "true": + self.success(f"Container {label} restarted and running (no healthcheck)") + return True + + self.console.print(f"[red]Container {label} failed to become healthy[/red]") + return False + def run_command( self, cmd: list[str], @@ -131,7 +207,7 @@ def create_progress(self, transient: bool = True) -> Progress: return Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), - console=self.console, + console=self.console.console, transient=transient, ) @@ -141,7 +217,7 @@ def success(self, message: str) -> None: Args: message: The message to print """ - self.console.print(f"[green]✅ {message}[/green]") + self.console.ok(message) def error(self, message: str) -> None: """Print an error message. @@ -149,7 +225,7 @@ def error(self, message: str) -> None: Args: message: The message to print """ - self.console.print(f"[red]❌ {message}[/red]") + self.console.error(message) def warning(self, message: str) -> None: """Print a warning message. @@ -157,7 +233,7 @@ def warning(self, message: str) -> None: Args: message: The message to print """ - self.console.print(f"[yellow]⚠️ {message}[/yellow]") + self.console.warn(message) def info(self, message: str) -> None: """Print an info message. @@ -165,7 +241,7 @@ def info(self, message: str) -> None: Args: message: The message to print """ - self.console.print(f"[blue]ℹ {message}[/blue]") + self.console.info(message) def ensure_data_directories( self, diff --git a/src/cli/deployment/constants.py b/src/cli/deployment/constants.py new file mode 100644 index 0000000..df33e2c --- /dev/null +++ b/src/cli/deployment/constants.py @@ -0,0 +1,13 @@ +"""Shared deployment constants.""" + +from pathlib import Path + +DEFAULT_DATA_SUBDIRS = [ + Path("postgres"), + Path("postgres-backups"), + Path("postgres-ssl"), + Path("redis"), + Path("redis-backups"), + Path("app-logs"), + Path("temporal-certs"), +] diff --git a/src/cli/deployment/dev_deployer.py b/src/cli/deployment/dev_deployer.py index 928ebe6..27158da 100644 --- a/src/cli/deployment/dev_deployer.py +++ b/src/cli/deployment/dev_deployer.py @@ -4,9 +4,9 @@ from typing import Any import typer -from rich.console import Console from rich.progress import Progress +from src.cli.shared.console import CLIConsole from src.dev.dev_utils import ( check_container_running, check_postgres_status, @@ -18,6 +18,7 @@ from src.utils.package_utils import get_package_module_path, get_package_root from .base import BaseDeployer +from .constants import DEFAULT_DATA_SUBDIRS from .health_checks import HealthChecker from .status_display import StatusDisplay @@ -26,17 +27,9 @@ class DevDeployer(BaseDeployer): """Deployer for development environment using Docker Compose.""" COMPOSE_FILE = "docker-compose.dev.yml" - DATA_SUBDIRS = [ - Path("postgres"), - Path("postgres-backups"), - Path("postgres-ssl"), - Path("redis"), - Path("redis-backups"), - Path("app-logs"), - Path("temporal-certs"), - ] - - def __init__(self, console: Console, project_root: Path): + DATA_SUBDIRS = DEFAULT_DATA_SUBDIRS + + def __init__(self, console: CLIConsole, project_root: Path): """Initialize the development deployer. Args: @@ -128,6 +121,32 @@ def show_status(self) -> None: """Display the current status of the development deployment.""" self.status_display.show_dev_status() + def deploy_secrets(self, **kwargs: Any) -> bool: + """Deploy secrets for Docker Compose development environment. + + For the development environment, secrets are hardcoded in + docker-compose.dev.yml for convenience. This is a no-op. + + Returns: + Always True (dev uses hardcoded credentials) + """ + self.info("Development environment uses hardcoded credentials") + self.info("No secret deployment needed") + return True + + def restart_resource( + self, label: str, resource_type: str, timeout: int = 120 + ) -> bool: + """Restart a Docker Compose container by name and wait for it to be healthy. + + Args: + label: Container name (e.g., 'api-forge-keycloak-dev', 'api-forge-postgres-dev') + + Returns: + True if restart succeeded and container is healthy, False otherwise + """ + return self.restart_container(label, self.health_checker, timeout) + def _get_running_services(self) -> list[str]: """Get list of currently running development services. diff --git a/src/cli/deployment/helm_deployer/__init__.py b/src/cli/deployment/helm_deployer/__init__.py index a47a78a..9061723 100644 --- a/src/cli/deployment/helm_deployer/__init__.py +++ b/src/cli/deployment/helm_deployer/__init__.py @@ -22,7 +22,6 @@ from .cleanup import CleanupManager from .config_sync import ConfigSynchronizer -from .constants import DeploymentConstants from .deployer import DeploymentError, HelmDeployer from .helm_release import HelmReleaseManager from .image_builder import ImageBuilder @@ -38,7 +37,6 @@ "ConfigSynchronizer", "HelmReleaseManager", "CleanupManager", - "DeploymentConstants", "DeploymentValidator", "ValidationResult", "ValidationSeverity", diff --git a/src/cli/deployment/helm_deployer/cleanup.py b/src/cli/deployment/helm_deployer/cleanup.py index f4983f9..71649fd 100644 --- a/src/cli/deployment/helm_deployer/cleanup.py +++ b/src/cli/deployment/helm_deployer/cleanup.py @@ -6,17 +6,15 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -from src.infra.k8s.controller import ReplicaSetInfo +from src.cli.shared.console import CLIConsole +from src.infra.constants import DeploymentConstants, DeploymentPaths +from src.infra.k8s.controller import KubernetesControllerSync, ReplicaSetInfo +from src.infra.k8s.helpers import get_k8s_controller_sync +from src.utils.paths import get_project_root from ..shell_commands import calculate_replicaset_age_hours -from .constants import DeploymentConstants - -if TYPE_CHECKING: - from rich.console import Console - from ..shell_commands import ShellCommands +CONTROLLER = get_k8s_controller_sync() class CleanupManager: @@ -30,8 +28,9 @@ class CleanupManager: def __init__( self, - commands: ShellCommands, - console: Console, + console: CLIConsole, + controller: KubernetesControllerSync = CONTROLLER, + paths: DeploymentPaths | None = None, constants: DeploymentConstants | None = None, ) -> None: """Initialize the cleanup manager. @@ -41,9 +40,10 @@ def __init__( console: Rich console for output constants: Optional deployment constants """ - self.commands = commands - self.console = console - self.constants = constants or DeploymentConstants() + self._console = console + self._constants = constants or DeploymentConstants() + self._paths = paths or DeploymentPaths(get_project_root()) + self._controller = controller def scale_down_old_replicasets(self, namespace: str) -> None: """Scale down old ReplicaSets to 0 replicas. @@ -55,25 +55,23 @@ def scale_down_old_replicasets(self, namespace: str) -> None: namespace: Target Kubernetes namespace """ try: - replicasets = self.commands.kubectl.get_replicasets(namespace) + replicasets = self._controller.get_replicasets(namespace) scaled_count = 0 for rs in replicasets: if not self._should_scale_down_replicaset(rs, namespace): continue - self.commands.kubectl.scale_replicaset(rs.name, namespace, 0) + self._controller.scale_replicaset(rs.name, namespace, 0) scaled_count += 1 if scaled_count > 0: - self.console.print( + self._console.print( f"[dim]✓ Scaled down {scaled_count} old ReplicaSet(s)[/dim]" ) except Exception as e: - self.console.print( - f"[dim yellow]⚠️ Could not scale down old ReplicaSets: {e}[/dim yellow]" - ) + self._console.warn(f"Could not scale down old ReplicaSets: {e}") def _should_scale_down_replicaset(self, rs: ReplicaSetInfo, namespace: str) -> bool: """Determine if a ReplicaSet should be scaled down. @@ -86,7 +84,7 @@ def _should_scale_down_replicaset(self, rs: ReplicaSetInfo, namespace: str) -> b True if the ReplicaSet is old and should be scaled down """ # Only process app/worker ReplicaSets with running pods - if not rs.name.startswith(self.constants.DEPLOYMENT_PREFIXES): + if not rs.name.startswith(self._constants.DEPLOYMENT_PREFIXES): return False if rs.replicas == 0: return False @@ -94,7 +92,7 @@ def _should_scale_down_replicaset(self, rs: ReplicaSetInfo, namespace: str) -> b return False # Check if this is an old revision - current_revision = self.commands.kubectl.get_deployment_revision( + current_revision = self._controller.get_deployment_revision( rs.owner_deployment, namespace ) return bool(rs.revision) and rs.revision != current_revision @@ -108,30 +106,26 @@ def cleanup_old_replicasets(self, namespace: str) -> None: Args: namespace: Target Kubernetes namespace """ - self.console.print("[bold cyan]🧹 Cleaning up old ReplicaSets...[/bold cyan]") + self._console.print("[bold cyan]🧹 Cleaning up old ReplicaSets...[/bold cyan]") try: - replicasets = self.commands.kubectl.get_replicasets(namespace) + replicasets = self._controller.get_replicasets(namespace) deleted_count = 0 for rs in replicasets: if not self._should_delete_replicaset(rs): continue - self.commands.kubectl.delete_replicaset(rs.name, namespace) + self._controller.delete_replicaset(rs.name, namespace) deleted_count += 1 if deleted_count > 0: - self.console.print( - f"[green]✓ Cleaned up {deleted_count} old ReplicaSet(s)[/green]" - ) + self._console.ok(f"✓ Cleaned up {deleted_count} old ReplicaSet(s)") else: - self.console.print("[dim]No old ReplicaSets to clean up[/dim]") + self._console.print("[dim]No old ReplicaSets to clean up[/dim]") except Exception as e: - self.console.print( - f"[yellow]⚠ Failed to clean up old ReplicaSets: {e}[/yellow]" - ) + self._console.warn(f"Failed to clean up old ReplicaSets: {e}") def _should_delete_replicaset(self, rs: ReplicaSetInfo) -> bool: """Determine if a ReplicaSet should be deleted. @@ -143,7 +137,7 @@ def _should_delete_replicaset(self, rs: ReplicaSetInfo) -> bool: True if the ReplicaSet is old enough to delete """ # Only delete app/worker ReplicaSets with 0 replicas - if not rs.name.startswith(self.constants.DEPLOYMENT_PREFIXES): + if not rs.name.startswith(self._constants.DEPLOYMENT_PREFIXES): return False if rs.replicas != 0: return False @@ -152,5 +146,5 @@ def _should_delete_replicaset(self, rs: ReplicaSetInfo) -> bool: age_hours = calculate_replicaset_age_hours(rs.created_at) return ( age_hours is not None - and age_hours > self.constants.REPLICASET_AGE_THRESHOLD_HOURS + and age_hours > self._constants.REPLICASET_AGE_THRESHOLD_HOURS ) diff --git a/src/cli/deployment/helm_deployer/config_sync.py b/src/cli/deployment/helm_deployer/config_sync.py index 9056da2..0bc6f24 100644 --- a/src/cli/deployment/helm_deployer/config_sync.py +++ b/src/cli/deployment/helm_deployer/config_sync.py @@ -16,11 +16,10 @@ from ruamel.yaml import YAML from src.app.runtime.config.config_loader import load_config - -from .constants import DeploymentPaths +from src.cli.shared.console import CLIConsole +from src.infra.constants import DeploymentPaths if TYPE_CHECKING: - from rich.console import Console from rich.progress import Progress @@ -35,7 +34,7 @@ class ConfigSynchronizer: def __init__( self, - console: Console, + console: CLIConsole, paths: DeploymentPaths, ) -> None: """Initialize the config synchronizer. @@ -61,15 +60,13 @@ def sync_config_to_values(self) -> None: values_path = self.paths.values_yaml if not config_path.exists(): - self.console.print( - "[yellow]⚠️ config.yaml not found, skipping sync[/yellow]" - ) + self.console.warn("config.yaml not found, skipping sync") return try: changes = self._compute_config_changes(config_path, values_path) if changes: - self.console.print("[green]✓ Synced changes:[/green]") + self.console.ok("Synced changes:") for change in changes: self.console.print(f" • {change}") else: @@ -77,7 +74,7 @@ def sync_config_to_values(self) -> None: "[dim] ✓ No changes needed (values already in sync)[/dim]" ) except Exception as e: - self.console.print(f"[yellow]⚠️ Failed to sync config: {e}[/yellow]") + self.console.warn(f"Failed to sync config: {e}") self.console.print("[dim] Continuing with existing values.yaml[/dim]") def _compute_config_changes( @@ -130,6 +127,17 @@ def _compute_config_changes( changes.append(f"temporal.enabled: {old_val} → {new_val}") modified = True + # Check postgres.enabled + if database_config := config_data.get("config", {}).get("database"): + if "bundled_postgres" in database_config: + if "postgres" in values_data: + old_val = values_data["postgres"].get("enabled", True) + new_val = database_config["bundled_postgres"].get("enabled", True) + if old_val != new_val: + values_data["postgres"]["enabled"] = new_val + changes.append(f"postgres.enabled: {old_val} → {new_val}") + modified = True + # Write back with preserved formatting if any changes were made if modified: with open(values_path, "w") as f: @@ -141,7 +149,8 @@ def copy_config_files(self, progress_factory: type[Progress]) -> None: """Copy configuration files to Helm staging area. Copies .env, config.yaml, PostgreSQL configs, Temporal scripts, - and entrypoint scripts to infra/helm/api-forge/files/. + and entrypoint scripts to infra/helm/api-forge/files/ and + infra/helm/api-forge-bundled-postgres/files/. Args: progress_factory: Rich Progress class for creating progress bars @@ -150,7 +159,10 @@ def copy_config_files(self, progress_factory: type[Progress]) -> None: "[bold cyan]📋 Copying config files to Helm staging area...[/bold cyan]" ) + # Prepare both destination directories self.paths.helm_files.mkdir(parents=True, exist_ok=True) + bundled_postgres_files = self.paths.postgres_standalone_chart / "files" + bundled_postgres_files.mkdir(parents=True, exist_ok=True) files_to_copy = self._get_config_files_manifest() @@ -159,18 +171,25 @@ def copy_config_files(self, progress_factory: type[Progress]) -> None: for source, dest_name, description in files_to_copy: if source.exists(): + # Copy to main api-forge chart dest_path = self.paths.helm_files / dest_name dest_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, dest_path) + + # Also copy to bundled-postgres chart + bundled_dest_path = bundled_postgres_files / dest_name + bundled_dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, bundled_dest_path) + self.console.print(f" [dim]✓ {description}[/dim]") else: - self.console.print( - f" [yellow]⚠ Skipped {description} (not found)[/yellow]" - ) + self.console.warn(f" Skipped {description} (not found)") progress.update(task, advance=1) rel_path = self.paths.helm_files.relative_to(self.paths.project_root) - self.console.print(f"[green]✓ Config files copied to {rel_path}[/green]") + bundled_rel_path = bundled_postgres_files.relative_to(self.paths.project_root) + self.console.ok(f"Config files copied to {rel_path}") + self.console.ok(f"Config files copied to {bundled_rel_path}") def _get_config_files_manifest(self) -> list[tuple[Path, str, str]]: """Get list of config files to copy to Helm staging. diff --git a/src/cli/deployment/helm_deployer/deployer.py b/src/cli/deployment/helm_deployer/deployer.py index 39da86a..a6b7633 100644 --- a/src/cli/deployment/helm_deployer/deployer.py +++ b/src/cli/deployment/helm_deployer/deployer.py @@ -14,19 +14,28 @@ from __future__ import annotations -from pathlib import Path from typing import Any -from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn +from src.cli.shared.console import CLIConsole, console +from src.infra.constants import ( + DEFAULT_CONSTANTS, + DEFAULT_PATHS, + DeploymentConstants, + DeploymentPaths, +) +from src.infra.k8s.controller import KubernetesControllerSync +from src.infra.k8s.helpers import get_k8s_controller_sync +from src.infra.k8s.port_forward import postgres_port_forward +from src.utils.paths import get_project_root + from ..base import BaseDeployer from ..health_checks import HealthChecker from ..shell_commands import ShellCommands from ..status_display import StatusDisplay from .cleanup import CleanupManager from .config_sync import ConfigSynchronizer -from .constants import DeploymentConstants, DeploymentPaths from .helm_release import HelmReleaseManager from .image_builder import DeploymentError, ImageBuilder from .secret_manager import SecretManager @@ -35,6 +44,9 @@ __all__ = ["HelmDeployer", "DeploymentError"] +CONTROLLER = get_k8s_controller_sync() + + class HelmDeployer(BaseDeployer): """Deployer for Kubernetes environment using Helm. @@ -63,18 +75,26 @@ class HelmDeployer(BaseDeployer): cleanup: Post-deployment cleanup manager """ - def __init__(self, console: Console, project_root: Path): + def __init__( + self, + console: CLIConsole, + controller: KubernetesControllerSync = CONTROLLER, + paths: DeploymentPaths | None = None, + constants: DeploymentConstants | None = None, + ) -> None: """Initialize the Kubernetes deployer. Args: console: Rich console for output project_root: Path to the project root directory """ + project_root = get_project_root() super().__init__(console, project_root) + self._controller = controller # Core configuration - self.constants = DeploymentConstants() - self.paths = DeploymentPaths(project_root) + self.constants = constants or DeploymentConstants() + self.paths = paths or DeploymentPaths(project_root) # UI components self.status_display = StatusDisplay(console) @@ -87,7 +107,7 @@ def __init__(self, console: Console, project_root: Path): self.image_builder = ImageBuilder( commands=self.commands, console=console, - project_root=project_root, + paths=self.paths, constants=self.constants, ) self.secret_manager = SecretManager( @@ -100,14 +120,14 @@ def __init__(self, console: Console, project_root: Path): paths=self.paths, ) self.helm_release = HelmReleaseManager( - commands=self.commands, console=console, + controller=self._controller, paths=self.paths, constants=self.constants, ) self.cleanup = CleanupManager( - commands=self.commands, console=console, + controller=self._controller, constants=self.constants, ) @@ -129,7 +149,7 @@ def _create_progress(self) -> Progress: return Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), - console=self.console, + console=self.console.console, transient=True, ) @@ -169,6 +189,7 @@ def deploy( ingress_tls_secret: str | None = None, ingress_tls_auto: bool = False, ingress_tls_staging: bool = False, + skip_db_check: bool = False, **kwargs: Any, ) -> None: """Deploy to Kubernetes cluster. @@ -176,13 +197,15 @@ def deploy( Deployment workflow: 1. Build Docker images with content-based tagging 2. Load images into target cluster (auto-detected) - 3. Generate and deploy secrets - 4. Sync config.yaml settings to values.yaml - 5. Copy config files to Helm staging area - 6. Deploy resources via Helm - 7. Restart deployments for consistency - 8. Wait for rollouts - 9. Clean up old resources + 3. Deploy secrets + 4. Restart postgres StatefulSet (if bundled postgres enabled) + 5. Verify database accessible (unless skip_db_check) + 6. Sync config.yaml settings to values.yaml + 7. Copy config files to Helm staging area + 8. Deploy resources via Helm + 9. Restart deployments for consistency + 10. Wait for rollouts + 11. Clean up old resources Args: namespace: Kubernetes namespace (default: api-forge-prod) @@ -192,6 +215,7 @@ def deploy( ingress_tls_secret: TLS secret name for HTTPS (manual) ingress_tls_auto: Auto-provision TLS via cert-manager ingress_tls_staging: Use staging Let's Encrypt (with ingress_tls_auto) + skip_db_check: Skip database verification before deployment **kwargs: Reserved for future options """ if not self.check_env_file(): @@ -246,6 +270,13 @@ def deploy( progress_factory=self._get_progress_class(), ) + # Phase 2.5: Restart postgres to pick up new secrets (if bundled postgres) + self._restart_postgres_if_needed(namespace) + + # Phase 2.6: Verify database is accessible after secret deployment + if not skip_db_check: + self._verify_database_or_exit(namespace) + # Phase 3: Prepare configuration self.config_sync.sync_config_to_values() self.config_sync.copy_config_files( @@ -277,14 +308,6 @@ def deploy( # Display final status self._show_deployment_success(namespace) - def _get_progress_class(self) -> type[Progress]: - """Get the Progress class for creating progress bars. - - Returns a class that can be instantiated to create progress bars. - This allows components to create their own progress instances. - """ - return Progress - def teardown(self, namespace: str | None = None, **kwargs: Any) -> None: """Remove Kubernetes deployment. @@ -301,7 +324,7 @@ def teardown(self, namespace: str | None = None, **kwargs: Any) -> None: self.commands.helm.uninstall(self.constants.HELM_RELEASE_NAME, namespace) with self.console.status(f"[bold red]Deleting namespace {namespace}..."): - self.commands.kubectl.delete_namespace(namespace, timeout="120s") + self._controller.delete_namespace(namespace, timeout="120s") self.success(f"Teardown complete for {namespace}") @@ -315,10 +338,96 @@ def show_status(self, namespace: str | None = None) -> None: namespace = self.constants.DEFAULT_NAMESPACE self.status_display.show_k8s_status(namespace) + def deploy_secrets(self, namespace: str | None = None) -> bool: + """Deploy Kubernetes secrets to the target namespace. + + Generates secrets if needed (first-time setup) and deploys them + to the specified Kubernetes namespace. + + Args: + namespace: Target Kubernetes namespace (default: api-forge-prod) + + Returns: + True if secrets were deployed successfully + """ + namespace = namespace or self.constants.DEFAULT_NAMESPACE + self.secret_manager.deploy_secrets( + namespace=namespace, + progress_factory=self._get_progress_class(), + ) + return True + + def restart_resource( + self, + label: str, + resource_type: str = "statefulset", + timeout: int = 300, + ) -> bool: + """Restart a Kubernetes resource and wait for it to be ready. + + Performs a rollout restart on the specified resource and blocks + until the rollout is complete and pods are ready. + + Args: + label: Resource name (e.g., 'postgres', 'app') + resource_type: Type of resource ('statefulset', 'deployment') + timeout: Maximum time to wait for rollout (e.g., '300s') + + Returns: + True if restart succeeded and resource is ready, False otherwise + """ + namespace = self.constants.DEFAULT_NAMESPACE + + self.console.print( + f"[bold cyan]♻️ Restarting {resource_type}/{label}...[/bold cyan]" + ) + + # Trigger the rollout restart + restart_result = self._controller.rollout_restart( + resource_type=resource_type, + namespace=namespace, + name=label, + ) + + if not restart_result.success: + self.console.print( + f"[red]Failed to restart {label}: {restart_result.stderr}[/red]" + ) + return False + + self.console.print(f"[green]✓ {label} restart initiated[/green]") + + # Wait for the rollout to complete + self.console.print(f"[dim]Waiting for {label} to be ready...[/dim]") + + wait_result = self._controller.rollout_status( + resource_type=resource_type, + namespace=namespace, + name=label, + timeout=f"{timeout}s", + ) + + if wait_result.success: + self.console.print(f"[green]✓ {label} is ready[/green]") + return True + else: + self.console.print( + f"[red]Timeout waiting for {label} to be ready: {wait_result.stderr}[/red]" + ) + return False + # ========================================================================= # Private Helpers # ========================================================================= + def _get_progress_class(self) -> type[Progress]: + """Get the Progress class for creating progress bars. + + Returns a class that can be instantiated to create progress bars. + This allows components to create their own progress instances. + """ + return Progress + def _show_deployment_success(self, namespace: str) -> None: """Display deployment success message and status. @@ -329,3 +438,198 @@ def _show_deployment_success(self, namespace: str) -> None: "\n[bold green]🎉 Kubernetes deployment complete![/bold green]" ) self.status_display.show_k8s_status(namespace) + + def _restart_postgres_if_needed(self, namespace: str) -> None: + """Restart postgres StatefulSet if it exists (for secret rotation). + + When secrets are rotated, the postgres pod needs to restart to pick up + new credentials and sync them to the database via the entrypoint script. + + Args: + namespace: Kubernetes namespace + """ + from src.infra.utils.service_config import is_bundled_postgres_enabled + + # Only restart if bundled postgres is enabled + if not is_bundled_postgres_enabled(): + return + + self.console.print( + "[bold cyan]♻️ Restarting postgres for secret sync...[/bold cyan]" + ) + + # Try to restart the StatefulSet (will fail gracefully if it doesn't exist) + restart_result = self._controller.rollout_restart( + "statefulset", namespace, self.constants.POSTGRES_RESOURCE_NAME + ) + + if restart_result.success: + self.console.print("[green]✓ Postgres restart triggered[/green]") + + # Wait for postgres to be ready + self.console.print("[dim]Waiting for postgres to be ready...[/dim]") + wait_result = self._controller.wait_for_pods( + namespace=namespace, + label_selector="app.kubernetes.io/name=postgres", + condition="ready", + timeout="300s", + ) + + if wait_result.success: + self.console.print("[green]✓ Postgres is ready[/green]") + # Wait additional time for password sync script to complete + # The postgres entrypoint syncs passwords AFTER postgres starts accepting connections + import time + + self.console.print( + "[dim]Waiting for password sync to complete (15s)...[/dim]" + ) + time.sleep(15) + else: + self.console.print( + "[yellow]⚠ Postgres may not be fully ready yet[/yellow]" + ) + else: + # StatefulSet doesn't exist or restart failed + # This is expected on first deployment before postgres is created + if "not found" in restart_result.stderr.lower(): + self.console.print( + "[dim] ℹ Postgres StatefulSet not found (will be created during deployment)[/dim]" + ) + else: + self.console.print( + f"[yellow]⚠ Postgres restart failed: {restart_result.stderr}[/yellow]" + ) + + def _verify_database_or_exit(self, namespace: str) -> None: + """Verify database is accessible or exit deployment. + + Uses port-forward to connect to the database and verify it's accessible. + Retries with exponential backoff to allow time for password sync after restart. + If verification fails after all retries, raises SystemExit to abort the deployment. + + Skips verification if postgres pod doesn't exist yet (first deployment). + + Args: + namespace: Kubernetes namespace + + Raises: + SystemExit: If database verification fails + """ + import time + + from src.infra.k8s import get_postgres_label + from src.infra.postgres.connection import get_settings + from src.infra.utils.service_config import is_bundled_postgres_enabled + + # Check if postgres pod exists first + postgres_label = get_postgres_label() + check_result = self._controller.wait_for_pods( + namespace=namespace, + label_selector=postgres_label, + condition="ready", + timeout="1s", # Just checking if it exists, not actually waiting + ) + + if not check_result.success: + # Postgres doesn't exist yet - this is a first deployment + self.console.print( + "[dim]ℹ️ Skipping database verification (postgres not yet deployed)[/dim]" + ) + return + + self.console.print( + "[bold cyan]🔍 Verifying database accessibility...[/bold cyan]" + ) + + max_retries = 10 + retry_delay = 5 # seconds + + def verify() -> bool: + for attempt in range(1, max_retries + 1): + try: + # Clear the settings cache to ensure fresh password values + # after secret rotation/deployment + get_settings.cache_clear() + settings = get_settings() + + # Import here to avoid circular dependencies + from src.infra.k8s.postgres_connection import ( + get_k8s_postgres_connection, + ) + + with postgres_port_forward( + namespace=namespace, pod_label=postgres_label + ): + conn = get_k8s_postgres_connection(settings) + success, msg = conn.test_connection() + + if success: + self.console.print( + f"[green]✅ Database accessible: {msg[:60]}...[/green]" + ) + return True + else: + if attempt < max_retries: + self.console.print( + f"[yellow]⚠ Attempt {attempt}/{max_retries} failed, " + f"retrying in {retry_delay}s (password sync may still be in progress)...[/yellow]" + ) + time.sleep(retry_delay) + else: + self.console.print( + f"[red]❌ Database check failed after {max_retries} attempts: {msg}[/red]" + ) + return False + + except ImportError: + # psycopg not installed, skip check + self.console.print( + "[yellow]⚠️ Database check skipped (psycopg not installed)[/yellow]" + ) + return True + except Exception as e: + if attempt < max_retries: + self.console.print( + f"[yellow]⚠ Attempt {attempt}/{max_retries} failed: {e}[/yellow]" + ) + self.console.print(f"[dim]Retrying in {retry_delay}s...[/dim]") + time.sleep(retry_delay) + else: + self.console.print( + f"[red]❌ Database check failed after {max_retries} attempts: {e}[/red]" + ) + return False + + return False + + # Run the verification with retries + if not verify(): + self.console.print( + "\n[red]❌ Database verification failed.[/red]\n" + "[dim]Please ensure PostgreSQL is running and accessible.[/dim]\n" + ) + + if is_bundled_postgres_enabled(): + self.console.print( + "[dim]For bundled PostgreSQL, ensure it was deployed:[/dim]\n" + " uv run api-forge-cli k8s db create\n" + " uv run api-forge-cli k8s db init\n" + ) + else: + self.console.print( + "[dim]For external PostgreSQL, verify DATABASE_URL in .env[/dim]\n" + ) + + raise SystemExit(1) + + +def get_deployer() -> HelmDeployer: + """Get the Helm deployer instance. + + Returns: + HelmDeployer instance configured for current project + """ + from src.cli.deployment.helm_deployer.deployer import HelmDeployer + + return HelmDeployer(console, CONTROLLER, DEFAULT_PATHS, DEFAULT_CONSTANTS) diff --git a/src/cli/deployment/helm_deployer/helm_release.py b/src/cli/deployment/helm_deployer/helm_release.py index e99f3e3..74a464b 100644 --- a/src/cli/deployment/helm_deployer/helm_release.py +++ b/src/cli/deployment/helm_deployer/helm_release.py @@ -8,16 +8,18 @@ import tempfile from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any import yaml # type: ignore[import-untyped] -from .constants import DeploymentConstants, DeploymentPaths +from src.cli.deployment.shell_commands import ShellCommands +from src.cli.shared.console import CLIConsole +from src.infra.constants import DeploymentConstants, DeploymentPaths +from src.infra.k8s.controller import KubernetesControllerSync +from src.infra.k8s.helpers import get_k8s_controller_sync +from src.utils.paths import get_project_root -if TYPE_CHECKING: - from rich.console import Console - - from ..shell_commands import ShellCommands +CONTROLLER = get_k8s_controller_sync() class HelmReleaseManager: @@ -33,9 +35,10 @@ class HelmReleaseManager: def __init__( self, - commands: ShellCommands, - console: Console, - paths: DeploymentPaths, + console: CLIConsole, + commands: ShellCommands | None = None, + controller: KubernetesControllerSync = CONTROLLER, + paths: DeploymentPaths | None = None, constants: DeploymentConstants | None = None, ) -> None: """Initialize the Helm release manager. @@ -46,10 +49,11 @@ def __init__( paths: Deployment path resolver constants: Optional deployment constants """ - self.commands = commands - self.console = console - self.paths = paths - self.constants = constants or DeploymentConstants() + self._commands = commands or ShellCommands(get_project_root()) + self._console = console + self._paths = paths or DeploymentPaths(get_project_root()) + self._constants = constants or DeploymentConstants() + self._controller = controller def create_image_override_file( self, @@ -77,15 +81,15 @@ def create_image_override_file( """ # Determine image repository - use registry prefix for remote clusters if registry: - app_repo = f"{registry}/{self.constants.APP_IMAGE_NAME}" - postgres_repo = f"{registry}/{self.constants.POSTGRES_IMAGE_NAME}" - redis_repo = f"{registry}/{self.constants.REDIS_IMAGE_NAME}" - temporal_repo = f"{registry}/{self.constants.TEMPORAL_IMAGE_NAME}" + app_repo = f"{registry}/{self._constants.APP_IMAGE_NAME}" + postgres_repo = f"{registry}/{self._constants.POSTGRES_IMAGE_NAME}" + redis_repo = f"{registry}/{self._constants.REDIS_IMAGE_NAME}" + temporal_repo = f"{registry}/{self._constants.TEMPORAL_IMAGE_NAME}" else: - app_repo = self.constants.APP_IMAGE_NAME - postgres_repo = self.constants.POSTGRES_IMAGE_NAME - redis_repo = self.constants.REDIS_IMAGE_NAME - temporal_repo = self.constants.TEMPORAL_IMAGE_NAME + app_repo = self._constants.APP_IMAGE_NAME + postgres_repo = self._constants.POSTGRES_IMAGE_NAME + redis_repo = self._constants.REDIS_IMAGE_NAME + temporal_repo = self._constants.TEMPORAL_IMAGE_NAME override_values: dict[str, Any] = { "app": {"image": {"repository": app_repo, "tag": image_tag}}, @@ -132,7 +136,7 @@ def create_image_override_file( override_values["app"]["ingress"] = ingress_config - self.console.print( + self._console.print( f"[bold cyan]🌐 Ingress enabled:[/bold cyan] {host}{tls_info}" ) @@ -140,7 +144,7 @@ def create_image_override_file( with open(temp_file, "w") as f: yaml.dump(override_values, f, default_flow_style=False) - self.console.print(f"[dim]Created image override file: {temp_file}[/dim]") + self._console.print(f"[dim]Created image override file: {temp_file}[/dim]") return temp_file def deploy_release( @@ -160,9 +164,9 @@ def deploy_release( from .image_builder import DeploymentError # Clean up any stuck releases first - self._cleanup_stuck_release(self.constants.HELM_RELEASE_NAME, namespace) + self._cleanup_stuck_release(self._constants.HELM_RELEASE_NAME, namespace) - self.console.print("[bold cyan]🚀 Deploying resources via Helm...[/bold cyan]") + self._console.print("[bold cyan]🚀 Deploying resources via Helm...[/bold cyan]") def print_helm_output(line: str) -> None: """Print Helm output in real-time, filtering noise.""" @@ -172,26 +176,26 @@ def print_helm_output(line: str) -> None: # Skip noisy warning lines about table values if "warning:" in line.lower() and "table" in line.lower(): return - self.console.print(f" [dim]{line}[/dim]") + self._console.print(f" [dim]{line}[/dim]") try: # Don't use --wait here. We need to restart all deployments first # so they pick up fresh secrets before waiting for pods to be ready. - result = self.commands.helm.upgrade_install( - release_name=self.constants.HELM_RELEASE_NAME, - chart_path=self.paths.helm_chart, + result = self._commands.helm.upgrade_install( + release_name=self._constants.HELM_RELEASE_NAME, + chart_path=self._paths.helm_chart, namespace=namespace, value_files=[image_override_file] if image_override_file.exists() else None, - timeout=self.constants.HELM_TIMEOUT, + timeout=self._constants.HELM_TIMEOUT, wait=False, wait_for_jobs=False, on_output=print_helm_output, ) if not result.success: - self.console.print("[red]✗ Helm deployment failed[/red]") + self._console.error("Helm deployment failed") raise DeploymentError( "Helm deployment failed", details=( @@ -214,11 +218,9 @@ def print_helm_output(line: str) -> None: # Clean up temporary override file if image_override_file.exists(): image_override_file.unlink() - self.console.print("[dim]Cleaned up temporary override file[/dim]") + self._console.print("[dim]Cleaned up temporary override file[/dim]") - self.console.print( - f"[green]✓ Helm manifests applied to namespace {namespace}[/green]" - ) + self._console.ok(f"Helm manifests applied to namespace {namespace}") def _cleanup_stuck_release(self, release_name: str, namespace: str) -> None: """Clean up Helm release stuck in problematic state. @@ -227,39 +229,34 @@ def _cleanup_stuck_release(self, release_name: str, namespace: str) -> None: release_name: Name of the Helm release namespace: Target namespace """ - stuck_releases = self.commands.helm.get_stuck_releases(namespace, release_name) + stuck_releases = self._commands.helm.get_stuck_releases(namespace, release_name) if not stuck_releases: return for release in stuck_releases: - self.console.print( - f"[yellow]⚠ Found release '{release.name}' in " - f"'{release.status}' state. Cleaning up...[/yellow]" + self._console.warn( + f"Found release '{release.name}' in " + f"'{release.status}' state. Cleaning up..." ) # Try normal uninstall first - result = self.commands.helm.uninstall(release.name, namespace) + result = self._commands.helm.uninstall(release.name, namespace) if result.success: - self.console.print( - f"[green]✓ Successfully cleaned up stuck release " - f"'{release.name}'[/green]" + self._console.ok( + f"Successfully cleaned up stuck release '{release.name}'" ) continue # Force cleanup if normal uninstall fails - self.console.print( - "[yellow]⚠ Normal uninstall failed. Attempting force cleanup...[/yellow]" - ) - self.commands.kubectl.delete_resources_by_label( + self._console.warn("Normal uninstall failed. Attempting force cleanup...") + self._controller.delete_resources_by_label( "all,configmap,secret,pvc", namespace, f"app.kubernetes.io/instance={release.name}", force=True, ) - self.commands.kubectl.delete_helm_secrets(namespace, release.name) - self.console.print( - f"[green]✓ Force cleaned up release '{release.name}'[/green]" - ) + self._controller.delete_helm_secrets(namespace, release.name) + self._console.ok(f"Force cleaned up release '{release.name}'") def restart_all_deployments(self, namespace: str) -> None: """Restart all deployments to pick up fresh secrets and configs. @@ -270,22 +267,21 @@ def restart_all_deployments(self, namespace: str) -> None: Args: namespace: Target Kubernetes namespace """ - self.console.print( + self._console.print( "[bold cyan]♻️ Restarting all deployments for consistency...[/bold cyan]" ) - result = self.commands.kubectl.rollout_restart("deployment", namespace) + result = self._controller.rollout_restart("deployment", namespace) if result.success: if result.stdout: for line in result.stdout.strip().split("\n"): - self.console.print(f" [dim]{line}[/dim]") - self.console.print("[green]✓ All deployments restarted[/green]") + self._console.print(f" [dim]{line}[/dim]") + self._console.ok("All deployments restarted successfully") else: - self.console.print( - f"[yellow]⚠ Rollout restart may have failed " - f"(exit code {result.returncode})[/yellow]" + self._console.warn( + f"Rollout restart may have failed (exit code {result.returncode})" ) if result.stderr: - self.console.print(f" [red]{result.stderr}[/red]") + self._console.error(f" {result.stderr}") def wait_for_rollouts(self, namespace: str) -> None: """Wait for all deployment rollouts to complete. @@ -293,37 +289,35 @@ def wait_for_rollouts(self, namespace: str) -> None: Args: namespace: Target Kubernetes namespace """ - self.console.print( + self._console.print( "[bold cyan]⏳ Waiting for rollouts to complete...[/bold cyan]" ) - deployments = self.commands.kubectl.get_deployments(namespace) + deployments = self._controller.get_deployments(namespace) if not deployments: - self.console.print("[yellow]⚠ No deployments found to wait for[/yellow]") + self._console.warn("No deployments found to wait for") return failed_deployments = [] for deployment in deployments: - with self.console.status(f"[cyan] Waiting for {deployment}...[/cyan]"): - result = self.commands.kubectl.rollout_status( + with self._console.status(f"[cyan] Waiting for {deployment}...[/cyan]"): + result = self._controller.rollout_status( "deployment", namespace, deployment, timeout="3m" ) if result.success: - self.console.print(f" [green]✓ {deployment} ready[/green]") + self._console.ok(f" {deployment} ready") else: - self.console.print(f" [yellow]⚠ {deployment} timed out[/yellow]") + self._console.warn(f" {deployment} timed out") failed_deployments.append(deployment) if failed_deployments: - self.console.print( - f"[yellow]⚠ Some rollouts timed out: {', '.join(failed_deployments)}[/yellow]" - ) - self.console.print( - f"[yellow]💡 Check status with: kubectl get pods -n {namespace}[/yellow]" + self._console.warn( + f"Some rollouts timed out: {', '.join(failed_deployments)}" ) - self.console.print( - f"[yellow]💡 To rollback: helm rollback " - f"{self.constants.HELM_RELEASE_NAME} -n {namespace}[/yellow]" + self._console.warn(f"💡 Check status with: kubectl get pods -n {namespace}") + self._console.warn( + f"💡 To rollback: helm rollback " + f"{self._constants.HELM_RELEASE_NAME} -n {namespace}" ) else: - self.console.print("[green]✓ All rollouts completed successfully[/green]") + self._console.ok("All rollouts completed successfully") diff --git a/src/cli/deployment/helm_deployer/image_builder.py b/src/cli/deployment/helm_deployer/image_builder.py index ec378fa..fe7796b 100644 --- a/src/cli/deployment/helm_deployer/image_builder.py +++ b/src/cli/deployment/helm_deployer/image_builder.py @@ -14,15 +14,21 @@ from pathlib import Path from typing import TYPE_CHECKING -from .constants import DeploymentConstants +from src.cli.shared.console import CLIConsole +from src.infra.constants import DeploymentConstants, DeploymentPaths +from src.infra.k8s import get_k8s_controller_sync +from src.infra.k8s.controller import KubernetesControllerSync +from src.utils.paths import get_project_root if TYPE_CHECKING: - from rich.console import Console from rich.progress import Progress from ..shell_commands import ShellCommands +CONTROLLER = get_k8s_controller_sync() + + class DeploymentError(Exception): """Raised when a deployment operation fails.""" @@ -48,9 +54,10 @@ class ImageBuilder: def __init__( self, + console: CLIConsole, commands: ShellCommands, - console: Console, - project_root: Path, + controller: KubernetesControllerSync = CONTROLLER, + paths: DeploymentPaths | None = None, constants: DeploymentConstants | None = None, ) -> None: """Initialize the image builder. @@ -61,10 +68,12 @@ def __init__( project_root: Path to project root constants: Optional deployment constants (uses defaults if not provided) """ - self.commands = commands - self.console = console - self.project_root = project_root - self.constants = constants or DeploymentConstants() + self._commands = commands + self._controller = controller + self._console = console + self._paths = paths or DeploymentPaths(get_project_root()) + self._constants = constants or DeploymentConstants() + self._project_root = self._paths.project_root def build_and_tag_images( self, @@ -85,14 +94,14 @@ def build_and_tag_images( Returns: Content-based image tag (e.g., "git-abc1234" or "hash-def5678") """ - self.console.print("[bold cyan]🔨 Building Docker images...[/bold cyan]") + self._console.print("[bold cyan]🔨 Building Docker images...[/bold cyan]") image_tag = self._generate_content_tag() - self.console.print(f"[dim]Using image tag: {image_tag}[/dim]") + self._console.print(f"[dim]Using image tag: {image_tag}[/dim]") # Skip rebuild only if ALL images exist with this tag if self._all_images_exist(image_tag): - self.console.print( + self._console.print( f"[yellow]✓ All images with tag {image_tag} already exist, " "skipping build[/yellow]" ) @@ -104,26 +113,26 @@ def build_and_tag_images( transient=True, ) as progress: task = progress.add_task("Building images...", total=1) - self.commands.docker.compose_build() + self._commands.docker.compose_build() progress.update(task, completed=1) # Tag app image with content-based tag - self.commands.docker.tag_image( - f"{self.constants.APP_IMAGE_NAME}:latest", - f"{self.constants.APP_IMAGE_NAME}:{image_tag}", + self._commands.docker.tag_image( + f"{self._constants.APP_IMAGE_NAME}:latest", + f"{self._constants.APP_IMAGE_NAME}:{image_tag}", ) # Tag infrastructure images with the same content-based tag # This ensures Kubernetes always pulls the correct version - for infra_image in self.constants.infra_image_names: - if self.commands.docker.image_exists(f"{infra_image}:latest"): - self.commands.docker.tag_image( + for infra_image in self._constants.infra_image_names: + if self._commands.docker.image_exists(f"{infra_image}:latest"): + self._commands.docker.tag_image( f"{infra_image}:latest", f"{infra_image}:{image_tag}", ) - self.console.print(f"[dim]Tagged {infra_image}:{image_tag}[/dim]") + self._console.print(f"[dim]Tagged {infra_image}:{image_tag}[/dim]") - self.console.print( + self._console.print( f"[green]✓ Docker images built and tagged: {image_tag}[/green]" ) @@ -142,12 +151,12 @@ def _generate_content_tag(self) -> str: Tag string (e.g., "git-a1b2c3d", "hash-123456789abc", "ts-1234567890") """ # Try git-based tag for clean repositories - git_status = self.commands.git.get_status() + git_status = self._commands.git.get_status() if git_status.is_git_repo and git_status.is_clean and git_status.short_sha: - self.console.print("[dim]✓ Clean git state, using commit SHA[/dim]") + self._console.print("[dim]✓ Clean git state, using commit SHA[/dim]") return f"git-{git_status.short_sha}" elif git_status.is_git_repo: - self.console.print( + self._console.print( "[dim]⚠ Uncommitted changes detected, using content hash[/dim]" ) @@ -184,7 +193,7 @@ def _compute_source_hash(self) -> str | None: files_hashed += 1 # Hash infrastructure files (Dockerfiles, scripts, configs) - infra_docker_dir = self.project_root / "infra" / "docker" / "prod" + infra_docker_dir = self._paths.docker_prod if infra_docker_dir.exists(): for infra_file in sorted(infra_docker_dir.rglob("*")): if infra_file.is_file() and self._is_infra_source_file(infra_file): @@ -193,7 +202,7 @@ def _compute_source_hash(self) -> str | None: # Also hash the main Dockerfile and docker-compose files for docker_file in ["Dockerfile", "docker-compose.prod.yml"]: - path = self.project_root / docker_file + path = self._project_root / docker_file if path.exists(): hasher.update(path.read_bytes()) files_hashed += 1 @@ -203,7 +212,7 @@ def _compute_source_hash(self) -> str | None: return hasher.hexdigest()[:12] except Exception as e: - self.console.print(f"[dim]Could not compute content hash: {e}[/dim]") + self._console.print(f"[dim]Could not compute content hash: {e}[/dim]") return None def _is_infra_source_file(self, path: Path) -> bool: @@ -252,7 +261,7 @@ def _find_package_directories(self) -> list[Path]: } packages = [] - for path in self.project_root.iterdir(): + for path in self._project_root.iterdir(): if path.is_dir() and path.name not in excluded_dirs: if (path / "__init__.py").exists(): packages.append(path) @@ -269,8 +278,8 @@ def _all_images_exist(self, image_tag: str) -> bool: """ all_images = self._get_all_images(image_tag) for image in all_images: - if not self.commands.docker.image_exists(image): - self.console.print(f"[dim]Image {image} not found, will rebuild[/dim]") + if not self._commands.docker.image_exists(image): + self._console.print(f"[dim]Image {image} not found, will rebuild[/dim]") return False return True @@ -283,10 +292,10 @@ def _get_all_images(self, image_tag: str) -> list[str]: Returns: List of fully qualified image names with tags """ - images = [f"{self.constants.APP_IMAGE_NAME}:{image_tag}"] + images = [f"{self._constants.APP_IMAGE_NAME}:{image_tag}"] # Use the same content-based tag for infra images to avoid stale image issues images.extend( - f"{name}:{image_tag}" for name in self.constants.infra_image_names + f"{name}:{image_tag}" for name in self._constants.infra_image_names ) return images @@ -309,10 +318,10 @@ def _load_images_to_cluster( registry: Container registry URL for remote clusters progress_factory: Rich Progress class for creating progress bars """ - context = self.commands.kubectl.get_current_context() + context = self._controller.get_current_context() # Determine cluster type and loading strategy - if self.commands.kubectl.is_minikube_context(): + if self._controller.is_minikube_context(): self._load_images_minikube(image_tag, progress_factory) elif "kind" in context.lower(): self._load_images_kind(image_tag, progress_factory) @@ -329,17 +338,17 @@ def _load_images_minikube( self, image_tag: str, progress_factory: type[Progress] ) -> None: """Load Docker images into Minikube's internal registry.""" - self.console.print("[bold cyan]📦 Loading images into Minikube...[/bold cyan]") + self._console.print("[bold cyan]📦 Loading images into Minikube...[/bold cyan]") images = self._get_all_images(image_tag) with progress_factory(transient=True) as progress: task = progress.add_task("Loading images...", total=len(images)) for image in images: - self.commands.docker.minikube_load_image(image) + self._commands.docker.minikube_load_image(image) progress.update(task, advance=1) - self.console.print( + self._console.print( f"[green]✓ Images loaded into Minikube with tag: {image_tag}[/green]" ) @@ -347,17 +356,17 @@ def _load_images_kind( self, image_tag: str, progress_factory: type[Progress] ) -> None: """Load Docker images into Kind cluster.""" - self.console.print("[bold cyan]📦 Loading images into Kind...[/bold cyan]") + self._console.print("[bold cyan]📦 Loading images into Kind...[/bold cyan]") images = self._get_all_images(image_tag) with progress_factory(transient=True) as progress: task = progress.add_task("Loading images...", total=len(images)) for image in images: - self.commands.docker.kind_load_image(image) + self._commands.docker.kind_load_image(image) progress.update(task, advance=1) - self.console.print( + self._console.print( f"[green]✓ Images loaded into Kind with tag: {image_tag}[/green]" ) @@ -368,7 +377,9 @@ def _push_images_to_registry( progress_factory: type[Progress], ) -> None: """Push Docker images to a remote container registry.""" - self.console.print(f"[bold cyan]📦 Pushing images to {registry}...[/bold cyan]") + self._console.print( + f"[bold cyan]📦 Pushing images to {registry}...[/bold cyan]" + ) # Build list of (local_image, remote_image) pairs local_images = self._get_all_images(image_tag) @@ -382,11 +393,11 @@ def _push_images_to_registry( for local_image, remote_image in image_pairs: # Tag for registry - self.commands.docker.tag_image(local_image, remote_image) + self._commands.docker.tag_image(local_image, remote_image) progress.update(task, advance=1) # Push to registry - self.commands.docker.push_image(remote_image) + self._commands.docker.push_image(remote_image) progress.update(task, advance=1) - self.console.print(f"[green]✓ Images pushed to {registry}[/green]") + self._console.print(f"[green]✓ Images pushed to {registry}[/green]") diff --git a/src/cli/deployment/helm_deployer/secret_manager.py b/src/cli/deployment/helm_deployer/secret_manager.py index d3a491e..2b3767f 100644 --- a/src/cli/deployment/helm_deployer/secret_manager.py +++ b/src/cli/deployment/helm_deployer/secret_manager.py @@ -8,10 +8,10 @@ from typing import TYPE_CHECKING -from .constants import DeploymentPaths +from src.cli.shared.console import CLIConsole +from src.infra.constants import DeploymentPaths if TYPE_CHECKING: - from rich.console import Console from rich.progress import Progress from ..shell_commands import ShellCommands @@ -29,7 +29,7 @@ class SecretManager: def __init__( self, commands: ShellCommands, - console: Console, + console: CLIConsole, paths: DeploymentPaths, ) -> None: """Initialize the secret manager. diff --git a/src/cli/deployment/helm_deployer/validator.py b/src/cli/deployment/helm_deployer/validator.py index d46c55e..75ce920 100644 --- a/src/cli/deployment/helm_deployer/validator.py +++ b/src/cli/deployment/helm_deployer/validator.py @@ -14,15 +14,18 @@ from dataclasses import dataclass, field from enum import Enum -from typing import TYPE_CHECKING -from src.infra.k8s.controller import PodInfo +from src.cli.deployment.shell_commands import ShellCommands +from src.cli.shared.console import CLIConsole +from src.infra.constants import DeploymentConstants, DeploymentPaths +from src.infra.k8s import ( + KubernetesControllerSync, + PodInfo, + get_k8s_controller_sync, +) +from src.utils.paths import get_project_root -if TYPE_CHECKING: - from rich.console import Console - - from ..shell_commands import ShellCommands - from .constants import DeploymentConstants +CONTROLLER = get_k8s_controller_sync() class ValidationSeverity(Enum): @@ -91,9 +94,11 @@ class DeploymentValidator: def __init__( self, + console: CLIConsole, commands: ShellCommands, - console: Console, - constants: DeploymentConstants, + controller: KubernetesControllerSync = CONTROLLER, + paths: DeploymentPaths | None = None, + constants: DeploymentConstants | None = None, ) -> None: """Initialize the validator. @@ -102,9 +107,11 @@ def __init__( console: Rich console for output constants: Deployment constants """ - self.commands = commands - self.console = console - self.constants = constants + self._console = console + self._commands = commands + self._controller = controller + self._paths = paths or DeploymentPaths(get_project_root()) + self._constants = constants or DeploymentConstants() def validate(self, namespace: str) -> ValidationResult: """Run all pre-deployment validation checks. @@ -143,25 +150,23 @@ def display_results(self, result: ValidationResult, namespace: str) -> None: """ if result.is_clean: if result.namespace_exists: - self.console.print("[dim]✓ Pre-deployment checks passed[/dim]") + self._console.print("[dim]✓ Pre-deployment checks passed[/dim]") return - self.console.print( - "\n[bold yellow]⚠️ Pre-deployment Issues Detected[/bold yellow]\n" - ) + self._console.warn("\nPre-deployment Issues Detected\n") for issue in result.issues: icon = self._get_severity_icon(issue.severity) color = self._get_severity_color(issue.severity) - self.console.print(f"[{color}]{icon} {issue.title}[/{color}]") - self.console.print(f" [dim]{issue.description}[/dim]") + self._console.print(f"[{color}]{icon} {issue.title}[/{color}]") + self._console.print(f" [dim]{issue.description}[/dim]") if issue.resource_name: - self.console.print( + self._console.print( f" [dim]Resource: {issue.resource_type}/{issue.resource_name}[/dim]" ) - self.console.print(f" [cyan]💡 {issue.recovery_hint}[/cyan]") - self.console.print() + self._console.print(f" [cyan]💡 {issue.recovery_hint}[/cyan]") + self._console.print() def prompt_cleanup(self, result: ValidationResult, namespace: str) -> bool: """Prompt user to clean up issues before proceeding. @@ -177,19 +182,19 @@ def prompt_cleanup(self, result: ValidationResult, namespace: str) -> bool: return True if result.requires_cleanup: - self.console.print( + self._console.print( "[bold red]Critical issues detected. Cleanup required before deployment.[/bold red]\n" ) - self.console.print( + self._console.print( "[yellow]Recommended: Run the following command to clean up:[/yellow]" ) - self.console.print( + self._console.print( "[bold cyan] uv run api-forge-cli deploy down k8s[/bold cyan]\n" ) - self.console.print( + self._console.print( "[dim]This will delete the Helm release and allow a fresh deployment.[/dim]" ) - self.console.print( + self._console.print( "[dim]Add --volumes only if you need to wipe persistent data (databases, etc).[/dim]\n" ) @@ -200,13 +205,11 @@ def prompt_cleanup(self, result: ValidationResult, namespace: str) -> bool: ) return response in ("y", "yes") except (KeyboardInterrupt, EOFError): - self.console.print("\n[dim]Deployment cancelled.[/dim]") + self._console.print("\n[dim]Deployment cancelled.[/dim]") return False elif result.has_errors: - self.console.print( - "[bold yellow]Errors detected that may cause deployment issues.[/bold yellow]\n" - ) + self._console.warn("Errors detected that may cause deployment issues.\n") # Prompt user try: @@ -215,12 +218,12 @@ def prompt_cleanup(self, result: ValidationResult, namespace: str) -> bool: ) return response in ("y", "yes") except (KeyboardInterrupt, EOFError): - self.console.print("\n[dim]Deployment cancelled.[/dim]") + self._console.print("\n[dim]Deployment cancelled.[/dim]") return False else: # Only warnings, continue with notification - self.console.print( + self._console.print( "[dim]Warnings detected but proceeding with deployment.[/dim]\n" ) return True @@ -234,45 +237,41 @@ def run_cleanup(self, namespace: str) -> bool: Returns: True if cleanup succeeded, False otherwise """ - self.console.print( + self._console.print( f"\n[bold red]🧹 Cleaning up namespace {namespace}...[/bold red]" ) try: # Uninstall Helm release first - helm_result = self.commands.helm.uninstall( - self.constants.HELM_RELEASE_NAME, namespace + helm_result = self._commands.helm.uninstall( + self._constants.HELM_RELEASE_NAME, namespace ) if helm_result.success: - self.console.print( - f"[green]✓ Helm release '{self.constants.HELM_RELEASE_NAME}' uninstalled[/green]" + self._console.ok( + f"Helm release '{self._constants.HELM_RELEASE_NAME}' uninstalled" ) else: - self.console.print( + self._console.print( "[dim]Helm release not found or already removed[/dim]" ) # Delete PVCs - pvc_result = self.commands.kubectl.delete_pvcs(namespace) + pvc_result = self._controller.delete_pvcs(namespace) if pvc_result.success: - self.console.print("[green]✓ Persistent volume claims deleted[/green]") + self._console.ok("Persistent volume claims deleted") # Delete namespace - ns_result = self.commands.kubectl.delete_namespace( - namespace, timeout="120s" - ) + ns_result = self._controller.delete_namespace(namespace, timeout="120s") if ns_result.success: - self.console.print(f"[green]✓ Namespace {namespace} deleted[/green]") + self._console.ok(f"Namespace {namespace} deleted") - self.console.print( - "\n[bold green]✓ Cleanup complete. You can now run deployment again.[/bold green]" - ) + self._console.ok("Cleanup complete. You can now run deployment again.") return True except Exception as e: - self.console.print(f"[red]✗ Cleanup failed: {e}[/red]") - self.console.print( - f"[yellow]💡 Try manual cleanup: kubectl delete namespace {namespace}[/yellow]" + self._console.error(f"Cleanup failed: {e}") + self._console.warn( + f"💡 Try manual cleanup: kubectl delete namespace {namespace}" ) return False @@ -282,13 +281,13 @@ def run_cleanup(self, namespace: str) -> bool: def _namespace_exists(self, namespace: str) -> bool: """Check if the namespace exists.""" - result = self.commands.kubectl.namespace_exists(namespace) + result = self._controller.namespace_exists(namespace) return result def _has_helm_release(self, namespace: str) -> bool: """Check if there's an existing Helm release.""" - releases = self.commands.helm.list_releases(namespace) - return any(r.name == self.constants.HELM_RELEASE_NAME for r in releases) + releases = self._commands.helm.list_releases(namespace) + return any(r.name == self._constants.HELM_RELEASE_NAME for r in releases) def _check_failed_jobs(self, namespace: str, result: ValidationResult) -> None: """Check for failed jobs in the namespace. @@ -301,7 +300,7 @@ def _check_failed_jobs(self, namespace: str, result: ValidationResult) -> None: them, and they may succeed on subsequent attempts as dependencies come online. """ - jobs = self.commands.kubectl.get_jobs(namespace) + jobs = self._controller.get_jobs(namespace) for job in jobs: job_name = job.name @@ -336,7 +335,7 @@ def _check_failed_jobs(self, namespace: str, result: ValidationResult) -> None: def _check_crashloop_pods(self, namespace: str, result: ValidationResult) -> None: """Check for pods in CrashLoopBackOff state.""" - pods = self.commands.kubectl.get_pods(namespace) + pods = self._controller.get_pods(namespace) for pod in pods: if pod.status == "CrashLoopBackOff": @@ -361,7 +360,7 @@ def _check_crashloop_pods(self, namespace: str, result: ValidationResult) -> Non def _check_pending_pods(self, namespace: str, result: ValidationResult) -> None: """Check for pods stuck in Pending state.""" - pods = self.commands.kubectl.get_pods(namespace) + pods = self._controller.get_pods(namespace) for pod in pods: if pod.status == "Pending": @@ -391,7 +390,7 @@ def _check_error_pods(self, namespace: str, result: ValidationResult) -> None: This avoids flagging old failed attempts when the job has since succeeded or has a newer attempt in progress. """ - pods = self.commands.kubectl.get_pods(namespace) + pods = self._controller.get_pods(namespace) # Group job-owned pods by their job name job_pods: dict[str, list[PodInfo]] = {} diff --git a/src/cli/deployment/prod_deployer.py b/src/cli/deployment/prod_deployer.py index 2bf2cb5..f4f9f5b 100644 --- a/src/cli/deployment/prod_deployer.py +++ b/src/cli/deployment/prod_deployer.py @@ -5,13 +5,16 @@ from typing import Any import typer -from rich.console import Console from rich.panel import Panel from rich.table import Table +from src.cli.shared.console import CLIConsole, console +from src.utils.paths import get_project_root + +from ...infra.utils.service_config import get_production_services from .base import BaseDeployer +from .constants import DEFAULT_DATA_SUBDIRS from .health_checks import HealthChecker -from .service_config import get_production_services from .status_display import StatusDisplay @@ -19,17 +22,9 @@ class ProdDeployer(BaseDeployer): """Deployer for production environment using Docker Compose.""" COMPOSE_FILE = "docker-compose.prod.yml" - DATA_SUBDIRS = [ - Path("postgres"), - Path("postgres-backups"), - Path("postgres-ssl"), - Path("redis"), - Path("redis-backups"), - Path("app-logs"), - Path("temporal-certs"), - ] - - def __init__(self, console: Console, project_root: Path): + DATA_SUBDIRS = DEFAULT_DATA_SUBDIRS + + def __init__(self, console: CLIConsole, project_root: Path): """Initialize the production deployer. Args: @@ -43,6 +38,10 @@ def __init__(self, console: Console, project_root: Path): # Build services list dynamically based on config.yaml self.SERVICES = get_production_services() + # ========================================================================= + # Public Interface + # ========================================================================= + def deploy(self, **kwargs: Any) -> None: """Deploy the production environment. @@ -87,116 +86,6 @@ def deploy(self, **kwargs: Any) -> None: ) self.status_display.show_prod_status() - def _ensure_required_directories(self) -> None: - data_root = self.ensure_data_directories(self.DATA_SUBDIRS) - self.info(f"Ensured data directories exist under {data_root}") - - def _validate_bind_mount_volumes(self) -> None: - """Validate bind-mount volumes and remove stale ones. - - Docker bind-mount volumes can become stale in two ways: - 1. The source directory was deleted while the volume metadata persists - 2. The bind mount references a deleted inode (shows as //deleted in findmnt) - - Both cases cause 'readdirent: no such file or directory' errors when trying - to start containers. - - This method detects stale bind-mount volumes and removes them so they can be - recreated fresh with the correct bind mount. - """ - project_name = "api-forge-prod" - - # Get list of volumes for this project - result = self.run_command( - [ - "docker", - "volume", - "ls", - "-q", - "--filter", - f"label=com.docker.compose.project={project_name}", - ], - capture_output=True, - ) - - if not result or not result.stdout: - return # No volumes to check - - volume_names = [ - v.strip() for v in result.stdout.strip().split("\n") if v.strip() - ] - stale_volumes = [] - - for volume_name in volume_names: - # Inspect the volume to check if it's a bind mount - inspect_result = self.run_command( - ["docker", "volume", "inspect", volume_name], - capture_output=True, - check=False, - ) - - if not inspect_result or not inspect_result.stdout: - continue - - try: - volume_info = json.loads(inspect_result.stdout) - if not volume_info: - continue - - options = volume_info[0].get("Options", {}) - mount_type = options.get("type", "") - bind_option = options.get("o", "") - device = options.get("device", "") - mountpoint = volume_info[0].get("Mountpoint", "") - - # Check if it's a bind mount - if mount_type == "none" and "bind" in bind_option and device: - # Check 1: Verify the source directory exists - source_path = Path(device) - if not source_path.exists(): - stale_volumes.append((volume_name, device, "missing")) - continue - - # Check 2: Verify the mount isn't pointing to a deleted inode - # This happens when rm -rf removes the directory while mount persists - if mountpoint: - findmnt_result = self.run_command( - ["findmnt", "-n", "-o", "SOURCE", mountpoint], - capture_output=True, - check=False, - ) - if findmnt_result and findmnt_result.stdout: - mount_source = findmnt_result.stdout.strip() - if "deleted" in mount_source.lower(): - stale_volumes.append( - (volume_name, device, "deleted inode") - ) - continue - - except (json.JSONDecodeError, KeyError, IndexError): - continue - - if stale_volumes: - self.console.print( - f"[yellow]⚠ Found {len(stale_volumes)} stale bind-mount volume(s)[/yellow]" - ) - for vol_name, device, reason in stale_volumes: - self.console.print(f" [dim]• {vol_name} → {device} ({reason})[/dim]") - - # Stop any containers using these volumes first - self.info("Stopping containers to remove stale volumes...") - self.teardown(volumes=False) - - # Remove stale volumes - for vol_name, _, _ in stale_volumes: - self.run_command( - ["docker", "volume", "rm", vol_name], - check=False, - capture_output=True, - ) - - self.success(f"Removed {len(stale_volumes)} stale volume(s)") - def teardown(self, **kwargs: Any) -> None: """Stop the production environment. @@ -327,6 +216,191 @@ def show_status(self) -> None: """Display the current status of the production deployment.""" self.status_display.show_prod_status() + def deploy_secrets(self) -> bool: + """Deploy secrets for Docker Compose environment. + + For Docker Compose, secrets are managed via Docker secrets mounted + from local files. This is a no-op as secrets are automatically + available when containers start via the secrets: section in + docker-compose.prod.yml. + + Returns: + Always True (secrets auto-mount from local files) + """ + self.info("Secrets are managed via Docker Compose secrets configuration") + self.info("No explicit deployment needed - secrets mount automatically") + return True + + def restart_resource( + self, label: str, resource_type: str, timeout: int = 120 + ) -> bool: + """Restart a Docker Compose container by name and wait for it to be healthy. + + Args: + label: Container name (e.g., 'api-forge-postgres', 'api-forge-app') + timeout: Maximum seconds to wait for container to be healthy + + Returns: + True if restart succeeded and container is healthy, False otherwise + """ + return self.restart_container(label, self.health_checker, timeout) + + # ========================================================================= + # Private Helpers + # ========================================================================= + + def _cleanup_stopped_containers(self) -> None: + """Remove stopped/stale containers from previous runs to avoid name conflicts. + + This handles containers from previous runs that may have been started + with a different docker-compose project name. + """ + # Forcibly remove any api-forge containers that aren't running + result = self.run_command( + [ + "docker", + "ps", + "-a", + "--filter", + "name=api-forge-", + "--format", + "{{.Names}}\t{{.State}}", + ], + capture_output=True, + check=False, + ) + + if result and result.stdout: + containers_to_remove = [] + for line in result.stdout.strip().split("\n"): + if not line.strip(): + continue + parts = line.strip().split("\t") + if len(parts) == 2: + name, state = parts + # Remove containers that aren't running + if state in ("created", "exited", "dead"): + containers_to_remove.append(name) + + if containers_to_remove: + self.info( + f"Removing {len(containers_to_remove)} stopped container(s): {', '.join(containers_to_remove)}" + ) + self.run_command( + ["docker", "rm", "-f"] + containers_to_remove, + check=False, + ) + + def _ensure_required_directories(self) -> None: + data_root = self.ensure_data_directories(self.DATA_SUBDIRS) + self.info(f"Ensured data directories exist under {data_root}") + + def _validate_bind_mount_volumes(self) -> None: + """Validate bind-mount volumes and remove stale ones. + + Docker bind-mount volumes can become stale in two ways: + 1. The source directory was deleted while the volume metadata persists + 2. The bind mount references a deleted inode (shows as //deleted in findmnt) + + Both cases cause 'readdirent: no such file or directory' errors when trying + to start containers. + + This method detects stale bind-mount volumes and removes them so they can be + recreated fresh with the correct bind mount. + """ + project_name = "api-forge-prod" + + # Get list of volumes for this project + result = self.run_command( + [ + "docker", + "volume", + "ls", + "-q", + "--filter", + f"label=com.docker.compose.project={project_name}", + ], + capture_output=True, + ) + + if not result or not result.stdout: + return # No volumes to check + + volume_names = [ + v.strip() for v in result.stdout.strip().split("\n") if v.strip() + ] + stale_volumes = [] + + for volume_name in volume_names: + # Inspect the volume to check if it's a bind mount + inspect_result = self.run_command( + ["docker", "volume", "inspect", volume_name], + capture_output=True, + check=False, + ) + + if not inspect_result or not inspect_result.stdout: + continue + + try: + volume_info = json.loads(inspect_result.stdout) + if not volume_info: + continue + + options = volume_info[0].get("Options", {}) + mount_type = options.get("type", "") + bind_option = options.get("o", "") + device = options.get("device", "") + mountpoint = volume_info[0].get("Mountpoint", "") + + # Check if it's a bind mount + if mount_type == "none" and "bind" in bind_option and device: + # Check 1: Verify the source directory exists + source_path = Path(device) + if not source_path.exists(): + stale_volumes.append((volume_name, device, "missing")) + continue + + # Check 2: Verify the mount isn't pointing to a deleted inode + # This happens when rm -rf removes the directory while mount persists + if mountpoint: + findmnt_result = self.run_command( + ["findmnt", "-n", "-o", "SOURCE", mountpoint], + capture_output=True, + check=False, + ) + if findmnt_result and findmnt_result.stdout: + mount_source = findmnt_result.stdout.strip() + if "deleted" in mount_source.lower(): + stale_volumes.append( + (volume_name, device, "deleted inode") + ) + continue + + except (json.JSONDecodeError, KeyError, IndexError): + continue + + if stale_volumes: + self.console.print( + f"[yellow]⚠ Found {len(stale_volumes)} stale bind-mount volume(s)[/yellow]" + ) + for vol_name, device, reason in stale_volumes: + self.console.print(f" [dim]• {vol_name} → {device} ({reason})[/dim]") + + # Stop any containers using these volumes first + self.info("Stopping containers to remove stale volumes...") + self.teardown(volumes=False) + + # Remove stale volumes + for vol_name, _, _ in stale_volumes: + self.run_command( + ["docker", "volume", "rm", vol_name], + check=False, + capture_output=True, + ) + + self.success(f"Removed {len(stale_volumes)} stale volume(s)") + def _build_app_image(self, force: bool = False) -> None: """Build the application Docker image using Docker layer caching. @@ -372,6 +446,9 @@ def _start_services(self, force_recreate: bool = False) -> None: Args: force_recreate: If True, force recreate containers (for secret rotation) """ + # Clean up any stopped/exited containers first to avoid name conflicts + self._cleanup_stopped_containers() + with self.create_progress() as progress: task = progress.add_task("Starting production services...", total=1) # Use fixed project name to avoid conflicts with old networks @@ -489,3 +566,13 @@ def check_health(name: str = container_name) -> bool: "Some services may need more time to become healthy. " "Check logs with: docker compose -f docker-compose.prod.yml logs [service]" ) + + +def get_deployer() -> ProdDeployer: + """Factory function to get the production deployer. + + + Returns: + An instance of ProdDeployer + """ + return ProdDeployer(console, get_project_root()) diff --git a/src/cli/deployment/shell_commands/__init__.py b/src/cli/deployment/shell_commands/__init__.py index af22f0a..25e085e 100644 --- a/src/cli/deployment/shell_commands/__init__.py +++ b/src/cli/deployment/shell_commands/__init__.py @@ -24,10 +24,11 @@ from pathlib import Path +from src.utils.paths import get_project_root + from .docker import DockerCommands from .git import GitCommands from .helm import HelmCommands -from .kubectl import KubectlCommands from .runner import CommandRunner from .types import ( CommandResult, @@ -57,20 +58,19 @@ class ShellCommands: ... commands.helm.upgrade_install("my-release", chart_path, namespace) """ - def __init__(self, project_root: Path) -> None: + def __init__(self, project_root: Path | None = None) -> None: """Initialize the shell commands executor. Args: project_root: Path to the project root directory. Commands will be executed from this directory by default. """ - self._project_root = Path(project_root) + self._project_root = Path(project_root) if project_root else get_project_root() self._runner = CommandRunner(self._project_root) # Initialize specialized command modules self.docker = DockerCommands(self._runner) self.helm = HelmCommands(self._runner) - self.kubectl = KubectlCommands(self._runner) self.git = GitCommands(self._runner) @property @@ -104,15 +104,6 @@ def docker_push_image(self, image_tag: str) -> CommandResult: """Push a Docker image. See docker.push_image.""" return self.docker.push_image(image_tag) - # Cluster detection - def is_minikube_context(self) -> bool: - """Check if using Minikube. See kubectl.is_minikube_context.""" - return self.kubectl.is_minikube_context() - - def get_current_context(self) -> str: - """Get current kubectl context. See kubectl.get_current_context.""" - return self.kubectl.get_current_context() - # Image loading def minikube_load_image(self, image_tag: str) -> CommandResult: """Load image into Minikube. See docker.minikube_load_image.""" @@ -141,54 +132,6 @@ def helm_get_stuck_releases(self, *args, **kwargs) -> list[HelmRelease]: # type """Get stuck Helm releases. See helm.get_stuck_releases.""" return self.helm.get_stuck_releases(*args, **kwargs) - # kubectl namespace/resource commands - def kubectl_delete_namespace(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Delete a namespace. See kubectl.delete_namespace.""" - return self.kubectl.delete_namespace(*args, **kwargs) - - def kubectl_delete_resources_by_label(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Delete resources by label. See kubectl.delete_resources_by_label.""" - return self.kubectl.delete_resources_by_label(*args, **kwargs) - - def kubectl_delete_helm_secrets(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Delete Helm secrets. See kubectl.delete_helm_secrets.""" - return self.kubectl.delete_helm_secrets(*args, **kwargs) - - # kubectl replicaset commands - def kubectl_get_replicasets(self, namespace: str) -> list[ReplicaSetInfo]: - """Get ReplicaSets. See kubectl.get_replicasets.""" - return self.kubectl.get_replicasets(namespace) - - def kubectl_delete_replicaset(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Delete a ReplicaSet. See kubectl.delete_replicaset.""" - return self.kubectl.delete_replicaset(*args, **kwargs) - - def kubectl_scale_replicaset(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Scale a ReplicaSet. See kubectl.scale_replicaset.""" - return self.kubectl.scale_replicaset(*args, **kwargs) - - # kubectl deployment commands - def kubectl_get_deployments(self, namespace: str) -> list[str]: - """Get deployments. See kubectl.get_deployments.""" - return self.kubectl.get_deployments(namespace) - - def kubectl_rollout_restart(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Restart a rollout. See kubectl.rollout_restart.""" - return self.kubectl.rollout_restart(*args, **kwargs) - - def kubectl_rollout_status(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Get rollout status. See kubectl.rollout_status.""" - return self.kubectl.rollout_status(*args, **kwargs) - - def kubectl_get_deployment_revision(self, *args, **kwargs) -> str | None: # type: ignore[no-untyped-def] - """Get deployment revision. See kubectl.get_deployment_revision.""" - return self.kubectl.get_deployment_revision(*args, **kwargs) - - # kubectl pod commands - def kubectl_wait_for_pods(self, *args, **kwargs) -> CommandResult: # type: ignore[no-untyped-def] - """Wait for pods. See kubectl.wait_for_pods.""" - return self.kubectl.wait_for_pods(*args, **kwargs) - # Git commands def git_get_status(self) -> GitStatus: """Get git status. See git.get_status.""" @@ -215,7 +158,6 @@ def run_bash_script( # Specialized command classes for direct usage "DockerCommands", "HelmCommands", - "KubectlCommands", "GitCommands", "CommandRunner", ] diff --git a/src/cli/deployment/shell_commands/docker.py b/src/cli/deployment/shell_commands/docker.py index 6cb07bc..ca6f226 100644 --- a/src/cli/deployment/shell_commands/docker.py +++ b/src/cli/deployment/shell_commands/docker.py @@ -98,8 +98,18 @@ def compose_build( >>> if result.success: ... print("Build complete") """ + from src.infra.utils.service_config import is_bundled_postgres_enabled + + # Determine which services to build based on configuration + services_to_build = ["app", "worker", "redis", "temporal", "temporal-web"] + + # Only build postgres if bundled postgres is enabled + # (postgres service is in a profile and causes dependency errors if not needed) + if is_bundled_postgres_enabled(): + services_to_build.append("postgres") + return self._runner.run( - ["docker", "compose", "-f", compose_file, "build"], + ["docker", "compose", "-f", compose_file, "build"] + services_to_build, capture_output=False, ) diff --git a/src/cli/deployment/shell_commands/kubectl.py b/src/cli/deployment/shell_commands/kubectl.py deleted file mode 100644 index 71a8070..0000000 --- a/src/cli/deployment/shell_commands/kubectl.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Kubectl command abstractions. - -This module provides commands for Kubernetes resource management via kubectl, -delegating to Kr8sController for the actual operations. - -This is a sync wrapper around the async Kr8sController for backward -compatibility with existing code. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from src.infra.k8s import Kr8sController, run_sync -from src.infra.k8s.controller import ( - CommandResult, - JobInfo, - PodInfo, - ReplicaSetInfo, -) - -if TYPE_CHECKING: - from .runner import CommandRunner - - -class KubectlCommands: - """Kubectl-related shell commands. - - This is a sync wrapper around Kr8sController that provides backward - compatibility with existing code. All methods delegate to the async - controller using run_sync(). - - Provides operations for: - - Cluster context detection - - Namespace management - - Deployment operations (rollout, restart, status) - - ReplicaSet management (list, scale, delete) - - Pod management (wait for conditions) - - Resource deletion by label - """ - - def __init__(self, runner: CommandRunner) -> None: - """Initialize kubectl commands. - - Args: - runner: Command runner (kept for interface compatibility, not used) - """ - # Keep runner reference for interface compatibility - self._runner = runner - # Delegate to the async controller - self._controller = Kr8sController() - - # ========================================================================= - # Cluster Context - # ========================================================================= - - def is_minikube_context(self) -> bool: - """Check if the current kubectl context is Minikube.""" - return run_sync(self._controller.is_minikube_context()) - - def get_current_context(self) -> str: - """Get the current kubectl context name.""" - return run_sync(self._controller.get_current_context()) - - # ========================================================================= - # Namespace Management - # ========================================================================= - - def namespace_exists(self, namespace: str) -> bool: - """Check if a namespace exists.""" - return run_sync(self._controller.namespace_exists(namespace)) - - def delete_namespace( - self, - namespace: str, - *, - wait: bool = True, - timeout: str = "120s", - ) -> CommandResult: - """Delete a Kubernetes namespace and all its resources.""" - return run_sync( - self._controller.delete_namespace(namespace, wait=wait, timeout=timeout) - ) - - def delete_pvcs(self, namespace: str) -> CommandResult: - """Delete all PersistentVolumeClaims in a namespace.""" - return run_sync(self._controller.delete_pvcs(namespace)) - - # ========================================================================= - # Resource Deletion - # ========================================================================= - - def delete_resources_by_label( - self, - resource_types: str, - namespace: str, - label_selector: str, - *, - force: bool = False, - ) -> CommandResult: - """Delete Kubernetes resources matching a label selector.""" - return run_sync( - self._controller.delete_resources_by_label( - resource_types, namespace, label_selector, force=force - ) - ) - - def delete_helm_secrets( - self, - namespace: str, - release_name: str, - ) -> CommandResult: - """Delete Helm release metadata secrets.""" - return run_sync(self._controller.delete_helm_secrets(namespace, release_name)) - - # ========================================================================= - # ReplicaSet Operations - # ========================================================================= - - def get_replicasets(self, namespace: str) -> list[ReplicaSetInfo]: - """Get all ReplicaSets in a namespace.""" - return run_sync(self._controller.get_replicasets(namespace)) - - def delete_replicaset( - self, - name: str, - namespace: str, - ) -> CommandResult: - """Delete a specific ReplicaSet.""" - return run_sync(self._controller.delete_replicaset(name, namespace)) - - def scale_replicaset( - self, - name: str, - namespace: str, - replicas: int, - ) -> CommandResult: - """Scale a ReplicaSet to a specific number of replicas.""" - return run_sync(self._controller.scale_replicaset(name, namespace, replicas)) - - # ========================================================================= - # Deployment Operations - # ========================================================================= - - def get_deployments(self, namespace: str) -> list[str]: - """Get list of deployment names in a namespace.""" - return run_sync(self._controller.get_deployments(namespace)) - - def rollout_restart( - self, - resource_type: str, - namespace: str, - name: str | None = None, - ) -> CommandResult: - """Trigger a rolling restart of a deployment/daemonset/statefulset.""" - return run_sync( - self._controller.rollout_restart(resource_type, namespace, name) - ) - - def rollout_status( - self, - resource_type: str, - namespace: str, - name: str | None = None, - *, - timeout: str = "300s", - ) -> CommandResult: - """Wait for a rollout to complete.""" - return run_sync( - self._controller.rollout_status( - resource_type, namespace, name, timeout=timeout - ) - ) - - def get_deployment_revision( - self, - name: str, - namespace: str, - ) -> str | None: - """Get the current revision number of a deployment.""" - return run_sync(self._controller.get_deployment_revision(name, namespace)) - - # ========================================================================= - # Pod Operations - # ========================================================================= - - def wait_for_pods( - self, - namespace: str, - label_selector: str, - *, - condition: str = "ready", - timeout: str = "300s", - ) -> CommandResult: - """Wait for pods matching a selector to reach a condition.""" - return run_sync( - self._controller.wait_for_pods( - namespace, label_selector, condition=condition, timeout=timeout - ) - ) - - def get_pods(self, namespace: str) -> list[PodInfo]: - """Get all pods in a namespace with their status. - - Note: Return type changed from list[dict] to list[PodInfo]. - Access fields as attributes: pod.name, pod.status, etc. - """ - return run_sync(self._controller.get_pods(namespace)) - - # ========================================================================= - # Job Operations - # ========================================================================= - - def get_jobs(self, namespace: str) -> list[JobInfo]: - """Get all jobs in a namespace with their status. - - Note: Return type changed from list[dict] to list[JobInfo]. - Access fields as attributes: job.name, job.status - """ - return run_sync(self._controller.get_jobs(namespace)) diff --git a/src/cli/deployment/status_display.py b/src/cli/deployment/status_display.py index 008de23..d170d98 100644 --- a/src/cli/deployment/status_display.py +++ b/src/cli/deployment/status_display.py @@ -4,9 +4,9 @@ import requests # type: ignore from dotenv.main import load_dotenv -from rich.console import Console from rich.panel import Panel +from src.cli.shared.console import CLIConsole from src.dev.dev_utils import ( check_container_running, check_docker_running, @@ -17,8 +17,8 @@ from src.infra.k8s import Kr8sController, run_sync from src.infra.k8s.controller import PodInfo, ServiceInfo +from ...infra.utils.service_config import get_production_services, is_temporal_enabled from .health_checks import HealthChecker -from .service_config import get_production_services, is_temporal_enabled # Module-level controller singleton _controller = Kr8sController() @@ -27,7 +27,7 @@ class StatusDisplay: """Utility class for displaying deployment status.""" - def __init__(self, console: Console): + def __init__(self, console: CLIConsole): """Initialize the status display. Args: @@ -118,23 +118,18 @@ def show_k8s_status(self, namespace: str = "api-forge-prod") -> None: def _show_docker_status(self) -> None: """Display Docker daemon status.""" - docker_running = check_docker_running() - status_text = ( - "[green]✅ Running[/green]" - if docker_running - else "[red]❌ Not running[/red]" - ) - self.console.print(f"Docker: {status_text}") + if check_docker_running(): + self.console.ok("Docker daemon is running") + else: + self.console.error("Docker daemon is not running") def _show_keycloak_status(self) -> None: """Display Keycloak status.""" keycloak_running = check_container_running("api-forge-keycloak-dev") - status_text = ( - "[green]✅ Running[/green]" - if keycloak_running - else "[red]❌ Not running[/red]" - ) - self.console.print(f"Keycloak: {status_text}") + if keycloak_running: + self.console.ok("Keycloak is running") + else: + self.console.error("Keycloak is not running") if keycloak_running: try: @@ -155,12 +150,10 @@ def _show_postgres_dev_status( ) -> None: """Display PostgreSQL development status.""" postgres_running = check_container_running("api-forge-postgres-dev") - status_text = ( - "[green]✅ Running[/green]" - if postgres_running - else "[red]❌ Not running[/red]" - ) - self.console.print(f"PostgreSQL: {status_text}") + if postgres_running: + self.console.ok("PostgreSQL is running") + else: + self.console.error("PostgreSQL is not running") if postgres_running and check_postgres_status(): self.console.print(" └─ Health: [green]✅ Ready[/green]") @@ -182,12 +175,10 @@ def _show_postgres_dev_status( def _show_redis_dev_status(self) -> None: """Display Redis development status.""" redis_running = check_container_running("api-forge-redis-dev") - status_text = ( - "[green]✅ Running[/green]" - if redis_running - else "[red]❌ Not running[/red]" - ) - self.console.print(f"Redis: {status_text}") + if redis_running: + self.console.ok("Redis is running") + else: + self.console.error("Redis is not running") if redis_running and check_redis_status(): self.console.print(" └─ Health: [green]✅ Ready[/green]") @@ -198,12 +189,10 @@ def _show_temporal_dev_status(self) -> None: temporal_server_running = check_container_running("api-forge-temporal-dev") temporal_web_running = check_container_running("api-forge-temporal-ui-dev") - status_text = ( - "[green]✅ Running[/green]" - if temporal_server_running - else "[red]❌ Not running[/red]" - ) - self.console.print(f"Temporal: {status_text}") + if temporal_server_running: + self.console.ok("Temporal Server is running") + else: + self.console.error("Temporal Server is not running") if temporal_server_running and check_temporal_status(): self.console.print(" └─ Health: [green]✅ Ready[/green]") diff --git a/src/cli/shared/__init__.py b/src/cli/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cli/shared/compose.py b/src/cli/shared/compose.py new file mode 100644 index 0000000..067e44b --- /dev/null +++ b/src/cli/shared/compose.py @@ -0,0 +1,79 @@ +"""Docker Compose command helpers.""" + +from __future__ import annotations + +import subprocess +from collections.abc import Sequence +from pathlib import Path + + +class ComposeRunner: + """Wrapper for Docker Compose commands with consistent defaults.""" + + def __init__( + self, + project_root: Path, + *, + compose_file: Path, + project_name: str | None = None, + ) -> None: + self._project_root = project_root + self._compose_file = compose_file + self._project_name = project_name + + def _base_cmd(self) -> list[str]: + cmd = ["docker", "compose"] + if self._project_name: + cmd.extend(["-p", self._project_name]) + cmd.extend(["-f", str(self._compose_file)]) + return cmd + + def run( + self, + args: Sequence[str], + *, + capture_output: bool = False, + check: bool = False, + ) -> subprocess.CompletedProcess[str]: + cmd = self._base_cmd() + list(args) + return subprocess.run( + cmd, + cwd=self._project_root, + capture_output=capture_output, + text=True, + check=check, + ) + + def logs( + self, + *, + service: str | None = None, + follow: bool = False, + tail: int | None = None, + ) -> subprocess.CompletedProcess[str]: + args = ["logs"] + if tail is not None: + args.append(f"--tail={tail}") + if follow: + args.append("--follow") + if service: + args.append(service) + return self.run(args, check=True) + + def restart( + self, *, service: str | None = None + ) -> subprocess.CompletedProcess[str]: + args = ["restart"] + if service: + args.append(service) + return self.run(args, check=True) + + def build( + self, *, service: str | None = None, no_cache: bool = False + ) -> subprocess.CompletedProcess[str]: + args = ["build"] + if no_cache: + args.append("--no-cache") + if service: + args.append(service) + return self.run(args, check=True) diff --git a/src/cli/shared/console.py b/src/cli/shared/console.py new file mode 100644 index 0000000..f0ef620 --- /dev/null +++ b/src/cli/shared/console.py @@ -0,0 +1,267 @@ +"""Shared utilities for CLI commands. + +This module provides common utilities used across all command modules, +including console output, confirmation dialogs, and path resolution. +""" + +from collections.abc import Callable +from typing import Literal + +import typer +from rich.console import Console, ConsoleRenderable +from rich.panel import Panel +from rich.status import Status + + +class CLIConsole: + """Rich console wrapper for consistent CLI output.""" + + def __init__(self) -> None: + """Initialize the CLI console.""" + self.console = Console() + + def print(self, msg: ConsoleRenderable | str | None = None) -> None: + self.console.print(msg) + + def status(self, status: str) -> Status: + return self.console.status(status) + + def info(self, msg: str) -> None: + self.console.print(f"[cyan]ℹ[/cyan] {msg}") + + def ok(self, msg: str) -> None: + self.console.print(f"[green]✅[/green] {msg}") + + def error(self, msg: str) -> None: + self.console.print(f"[red]❌[/red] {msg}") + + def warn(self, msg: str) -> None: + self.console.print(f"[yellow]⚠️[/yellow] {msg}") + + def confirm_action( + self, + action: str, + details: str | None = None, + extra_warning: str | None = None, + force: bool = False, + ) -> bool: + """Prompt user to confirm a potentially destructive action. + + Args: + action: Description of the action (e.g., "Stop all services") + details: Additional details about what will be affected + extra_warning: Extra warning message (e.g., for data loss) + force: If True, skip the confirmation prompt + + Returns: + True if the user confirmed, False otherwise + """ + if force: + return True + + # Build warning message + warning_lines = [f"[bold red]⚠️ {action}[/bold red]"] + + if details: + warning_lines.append(f"\n{details}") + + if extra_warning: + warning_lines.append(f"\n[yellow]{extra_warning}[/yellow]") + + self.console.print( + Panel( + "\n".join(warning_lines), + title="Confirmation Required", + border_style="red", + ) + ) + + try: + response = self.console.input( + "\n[bold]Are you sure you want to proceed?[/bold] \\[y/N]: " + ) + return response.strip().lower() in ("y", "yes") + except (KeyboardInterrupt, EOFError): + self.console.print("\n[dim]Cancelled.[/dim]") + return False + + def handle_error( + self, message: str, details: str | None = None, exit_code: int = 1 + ) -> None: + """Handle an error by printing a message and exiting. + + Args: + message: Error message to display + details: Optional additional details + exit_code: Exit code to use + """ + self.error(f"\n[bold red]❌ {message}[/bold red]\n") + if details: + self.console.print(Panel(details, title="Details", border_style="red")) + raise typer.Exit(exit_code) + + def prompt_choice( + self, + title: str, + choices: list[tuple[str, str]], + *, + default: int = 1, + cancel_option: bool = True, + ) -> int: + """Prompt user to select from numbered choices. + + Args: + title: Title/question to display + choices: List of (short_name, description) tuples + default: Default choice (1-indexed) + cancel_option: Whether to add a cancel option + + Returns: + Selected choice number (1-indexed), or 0 if cancelled + """ + self.console.print(f"\n[yellow]{title}[/yellow]\n") + + # Display numbered options + for i, (name, description) in enumerate(choices, 1): + marker = "[bold cyan]→[/bold cyan]" if i == default else " " + self.console.print(f" {marker} [bold]{i}.[/bold] {name}") + if description: + self.console.print(f" [dim]{description}[/dim]") + + if cancel_option: + self.console.print(" [bold]0.[/bold] Cancel") + + try: + while True: + response = self.console.input(f"\nEnter choice [{default}]: ").strip() + + if not response: + return default + + if response == "0" and cancel_option: + self.console.print("[dim]Cancelled.[/dim]") + return 0 + + try: + choice = int(response) + if 1 <= choice <= len(choices): + return choice + self.console.print( + f"[red]Please enter a number between 1 and {len(choices)}[/red]" + ) + except ValueError: + self.console.print("[red]Please enter a valid number[/red]") + + except (KeyboardInterrupt, EOFError): + self.console.print("\n[dim]Cancelled.[/dim]") + return 0 + + def prompt_resource_conflict( + self, + resource_type: str, + resource_name: str, + namespace: str, + *, + reason: str = "immutable fields changed", + data_warning: bool = True, + ) -> Literal["recreate", "skip", "cancel"]: + """Prompt user to handle a Kubernetes resource update conflict. + + This handles cases where a resource (StatefulSet, etc.) cannot be + updated in-place due to immutable field changes. + + Args: + resource_type: Type of resource (e.g., "StatefulSet", "Deployment") + resource_name: Name of the resource + namespace: Kubernetes namespace + reason: Why the update failed + data_warning: Whether to warn about potential data loss + + Returns: + One of: "recreate", "skip", or "cancel" + """ + self.warn(f"{resource_type} update failed - {reason}") + self.console.print( + f"\n[dim]{resource_type}s have immutable fields that cannot be updated in-place.[/dim]" + ) + self.console.print( + "[dim]To apply these changes, the resource must be deleted and recreated.[/dim]" + ) + + choices = [ + ( + "Delete and recreate", + "PVCs retained, but Helm release history will be reset" + if data_warning + else "Resource will be recreated, Helm history reset", + ), + ("Skip", f"Keep existing {resource_type.lower()}"), + ] + + choice = self.prompt_choice( + f"How would you like to proceed with {resource_name}?", + choices, + default=1, # Default to recreate (user's intent) + cancel_option=True, + ) + + if choice == 1: + return "recreate" + elif choice == 2: + return "skip" + else: + return "cancel" + + def print_header(self, title: str, style: str = "blue") -> None: + """Print a styled header panel. + + Args: + title: Header title text + style: Border style color + """ + self.console.print( + Panel.fit( + f"[bold {style}]{title}[/bold {style}]", + border_style=style, + ) + ) + + def print_subheader(self, title: str) -> None: + """Print a subheader. + + Args: + title: Subheader title text + """ + self.console.print(f"\n[bold underline]{title}[/bold underline]\n") + + +def with_error_handling(func: Callable[..., None]) -> Callable[..., None]: + """Decorator to wrap command functions with standard error handling. + + Catches common exceptions and formats them consistently. + + Args: + func: The command function to wrap + + Returns: + Wrapped function with error handling + """ + from functools import wraps + + from src.cli.deployment.helm_deployer.image_builder import DeploymentError + + @wraps(func) + def wrapper(*args: object, **kwargs: object) -> None: + try: + func(*args, **kwargs) + except DeploymentError as e: + console.handle_error(e.message, e.details) + except KeyboardInterrupt: + console.print("\n[dim]Operation cancelled by user.[/dim]") + raise typer.Exit(130) from None + + return wrapper + + +# Shared console instance for consistent output +console = CLIConsole() diff --git a/src/cli/shared/secrets.py b/src/cli/shared/secrets.py new file mode 100644 index 0000000..c2be5fa --- /dev/null +++ b/src/cli/shared/secrets.py @@ -0,0 +1,29 @@ +"""Utility functions for handling secrets.""" + +import getpass +import os + +import typer + +from src.cli.shared.console import console + + +def get_password(prompt: str, env_var: str | None = None) -> str: + """Get password from environment or prompt user.""" + if env_var: + password = os.environ.get(env_var) + if password: + console.print(f"[dim]Using password from {env_var}[/dim]") + return password + else: + console.print(f"[dim]Environment variable {env_var} not set[/dim]") + + try: + password = getpass.getpass(prompt) + if not password: + console.print("[red]❌ No password entered[/red]") + raise typer.Exit(1) from None + return password + except (EOFError, KeyboardInterrupt): + console.print("\n[dim]Password input cancelled[/dim]") + raise typer.Exit(1) from None diff --git a/src/cli/deployment/helm_deployer/constants.py b/src/infra/constants.py similarity index 70% rename from src/cli/deployment/helm_deployer/constants.py rename to src/infra/constants.py index 7cb7bdc..7737f24 100644 --- a/src/cli/deployment/helm_deployer/constants.py +++ b/src/infra/constants.py @@ -10,6 +10,8 @@ from dataclasses import dataclass from pathlib import Path +from src.utils.paths import get_project_root + @dataclass(frozen=True) class DeploymentConstants: @@ -25,6 +27,7 @@ class DeploymentConstants: DEFAULT_NAMESPACE: str = "api-forge-prod" HELM_RELEASE_NAME: str = "api-forge" HELM_CHART_NAME: str = "api-forge" + POSTGRES_POD_LABEL: str = "app.kubernetes.io/name=postgres" # Timeouts HELM_TIMEOUT: str = "10m" @@ -40,6 +43,12 @@ class DeploymentConstants: REDIS_IMAGE_NAME: str = "app_data_redis_image" TEMPORAL_IMAGE_NAME: str = "my-temporal-server" + # Resource names (used for both Docker Compose and Kubernetes) + APP_RESOURCE_NAME: str = "api-forge-app" + POSTGRES_RESOURCE_NAME: str = "api-forge-postgres" + REDIS_RESOURCE_NAME: str = "api-forge-redis" + TEMPORAL_RESOURCE_NAME: str = "api-forge-temporal" + # Infrastructure images (tagged with same content-based tag as app) @property def infra_image_names(self) -> tuple[str, ...]: @@ -57,6 +66,9 @@ def infra_image_names(self) -> tuple[str, ...]: DOCKER_DIR: str = "docker" DOCKER_PROD_DIR: str = "prod" + # Default random ephemeral port for connection forwarding + DEFAULT_EPHEMERAL_PORT: int = 54320 + # Registry URL validation pattern # Matches: host.com/path, host:port/path, localhost:5000 REGISTRY_PATTERN: re.Pattern[str] = re.compile( @@ -77,8 +89,8 @@ def __init__(self, project_root: Path) -> None: Args: project_root: Path to the project root directory """ - self.project_root = project_root - self._constants = DeploymentConstants() + self._project_root = project_root + self._constants = DEFAULT_CONSTANTS # Build derived paths self.infra = project_root / self._constants.INFRA_DIR @@ -92,6 +104,26 @@ def __init__(self, project_root: Path) -> None: self.infra / self._constants.DOCKER_DIR / self._constants.DOCKER_PROD_DIR ) + @property + def project_root(self) -> Path: + """Get path to project root.""" + return self._project_root + + @property + def templates_dir(self) -> Path: + """Get path to Helm templates directory.""" + return self.helm_chart / "templates" + + @property + def postgres_bundled_chart_yaml(self) -> Path: + """Get path to PostgreSQL StatefulSet YAML (in main chart).""" + return self.templates_dir / "statefulsets" / "postgres.yaml" + + @property + def postgres_standalone_chart(self) -> Path: + """Get path to standalone bundled PostgreSQL Helm chart directory.""" + return self.infra / self._constants.HELM_DIR / "api-forge-bundled-postgres" + @property def config_yaml(self) -> Path: """Get path to config.yaml.""" @@ -102,6 +134,11 @@ def values_yaml(self) -> Path: """Get path to Helm values.yaml.""" return self.helm_chart / "values.yaml" + @property + def bitnami_postgres_values_yaml(self) -> Path: + """Get path to Bitnami PostgreSQL Helm values.yaml.""" + return self.helm_chart / "values-bitnami-postgres.yaml" + @property def env_file(self) -> Path: """Get path to .env file.""" @@ -121,3 +158,7 @@ def generate_secrets_script(self) -> Path: def secrets_keys_dir(self) -> Path: """Get path to secrets keys directory.""" return self.secrets / "keys" + + +DEFAULT_CONSTANTS = DeploymentConstants() +DEFAULT_PATHS = DeploymentPaths(project_root=get_project_root()) diff --git a/src/infra/docker_compose/__init__.py b/src/infra/docker_compose/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/infra/docker_compose/postgres_connection.py b/src/infra/docker_compose/postgres_connection.py new file mode 100644 index 0000000..c09d0b0 --- /dev/null +++ b/src/infra/docker_compose/postgres_connection.py @@ -0,0 +1,74 @@ +from typing import Any, override + +from src.infra.postgres.connection import DbSettings, PostgresConnection +from src.infra.utils.service_config import is_bundled_postgres_enabled + + +class DockerComposePostgresConnectionBundled(PostgresConnection): + def __init__( + self, + settings: DbSettings, + superuser_mode: bool = False, + ) -> None: + """PostgreSQL connection manager for bundled Postgres. + Args: + settings: Database settings + user: Override user (default: settings.user) + password: Override password (default: settings.password) + """ + super().__init__( + settings, + superuser_mode=superuser_mode, + ssl_mode="require", + ) + + @override + def get_dsn(self, database: str | None = None) -> dict[str, Any]: + """Get connection parameters for psycopg2.connect(). + + Args: + database: Override database name + + Returns: + Dict of connection parameters + """ + return { + "host": "localhost", + "port": self._settings.port, + "dbname": database or self._settings.app_db, + "user": self._superuser_mode + and self._settings.superuser + or self._settings.user, + "password": self._superuser_mode + and self._settings.superuser_password + or self._settings.password, + "sslmode": self._ssl_mode, + } + + +class DockerComposePostgresConnection(PostgresConnection): ... + + +def get_docker_compose_postgres_connection( + settings: DbSettings, + superuser_mode: bool = False, +) -> PostgresConnection: + """Get a Docker Compose PostgreSQL connection. + Args: + settings: Database settings + superuser_mode: Whether to connect as superuser + bundled_postgres: Whether to use bundled Postgres (port-forward) + + Returns: + PostgreSQL connection + """ + if is_bundled_postgres_enabled(): + return DockerComposePostgresConnectionBundled( + settings=settings, + superuser_mode=superuser_mode, + ) + else: + return DockerComposePostgresConnection( + settings=settings, + superuser_mode=superuser_mode, + ) diff --git a/src/infra/k8s/__init__.py b/src/infra/k8s/__init__.py index 25a7c5e..2b3263a 100644 --- a/src/infra/k8s/__init__.py +++ b/src/infra/k8s/__init__.py @@ -20,22 +20,32 @@ pods = run_sync(kr8s_controller.get_pods("my-namespace")) """ +from ..constants import DeploymentConstants, DeploymentPaths from .controller import ( ClusterIssuerStatus, CommandResult, JobInfo, KubernetesController, + KubernetesControllerSync, PodInfo, ReplicaSetInfo, ServiceInfo, ) +from .helpers import ( + get_k8s_controller, + get_k8s_controller_sync, + get_namespace, + get_postgres_label, +) from .kr8s_controller import Kr8sController from .kubectl_controller import KubectlController +from .port_forward import with_postgres_port_forward from .utils import run_sync __all__ = [ # Controller classes "KubernetesController", + "KubernetesControllerSync", "KubectlController", "Kr8sController", # Data classes @@ -47,4 +57,12 @@ "ClusterIssuerStatus", # Utilities "run_sync", + "with_postgres_port_forward", + "get_k8s_controller", + "get_k8s_controller_sync", + "get_namespace", + "get_postgres_label", + # Constants + "DeploymentConstants", + "DeploymentPaths", ] diff --git a/src/infra/k8s/controller.py b/src/infra/k8s/controller.py index 6997eff..eef50ed 100644 --- a/src/infra/k8s/controller.py +++ b/src/infra/k8s/controller.py @@ -10,6 +10,9 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path +from typing import Any + +from src.infra.k8s.utils import run_sync # ============================================================================= # Data Types @@ -184,6 +187,52 @@ async def apply_manifest(self, manifest_path: Path) -> CommandResult: """ ... + @abstractmethod + async def resource_exists( + self, + resource_type: str, + name: str, + namespace: str, + ) -> bool: + """Check if a Kubernetes resource exists. + + Args: + resource_type: Resource type (e.g., "statefulset", "deployment", "pod") + name: Resource name + namespace: Kubernetes namespace + + Returns: + True if the resource exists, False otherwise + """ + ... + + @abstractmethod + async def delete_resource( + self, + resource_type: str, + name: str, + namespace: str, + *, + cascade: str | None = None, + wait: bool = True, + ) -> CommandResult: + """Delete a specific Kubernetes resource by name. + + Args: + resource_type: Resource type (e.g., "statefulset", "deployment", "pod") + name: Resource name + namespace: Kubernetes namespace + cascade: Cascade deletion policy (None uses k8s default): + - "background": Delete dependents in background + - "foreground": Delete dependents in foreground + - "orphan": Leave dependents running (don't delete) + wait: Whether to wait for deletion to complete + + Returns: + CommandResult with deletion status + """ + ... + @abstractmethod async def delete_resources_by_label( self, @@ -192,6 +241,7 @@ async def delete_resources_by_label( label_selector: str, *, force: bool = False, + cascade: str | None = None, ) -> CommandResult: """Delete Kubernetes resources matching a label selector. @@ -202,6 +252,10 @@ async def delete_resources_by_label( label_selector: Label selector (e.g., "app.kubernetes.io/instance=my-app") force: Whether to force delete (bypass graceful deletion) + cascade: Cascade deletion policy (None uses k8s default): + - "background": Delete dependents in background + - "foreground": Delete dependents in foreground + - "orphan": Leave dependents running (don't delete) Returns: CommandResult with deletion status @@ -360,11 +414,16 @@ async def scale_replicaset( # ========================================================================= @abstractmethod - async def get_pods(self, namespace: str) -> list[PodInfo]: + async def get_pods( + self, + namespace: str, + label_selector: str | None = None, + ) -> list[PodInfo]: """Get all pods in a namespace with their status. Args: namespace: Kubernetes namespace + label_selector: Optional label selector to filter pods (e.g., "app=postgres") Returns: List of PodInfo objects with pod details @@ -492,3 +551,312 @@ async def get_cluster_issuer_yaml(self, issuer_name: str) -> str | None: YAML string, or None if not found """ ... + + +class KubernetesControllerSync: + """Synchronous wrapper for KubernetesController. + + Automatically wraps all async methods from the underlying controller + and exposes them as synchronous methods using run_sync(). + """ + + def __init__(self, controller: KubernetesController): + self._controller = controller + + def __getattr__(self, name: str) -> Any: + """Dynamically wrap async methods as sync.""" + attr = getattr(self._controller, name) + + # If it's a coroutine function, wrap it + if callable(attr) and hasattr(attr, "__code__"): + import inspect + + if inspect.iscoroutinefunction(attr): + + def sync_wrapper(*args: Any, **kwargs: Any) -> Any: + return run_sync(attr(*args, **kwargs)) + + return sync_wrapper + + # Otherwise return as-is (properties, non-async methods) + return attr + + +# ============================================================================= +# Stub File Generation +# ============================================================================= + + +def generate_sync_stubs() -> str: + """Generate a .pyi stub file using AST parsing. + + This function automatically extracts dataclass definitions and method signatures + from the source file using AST parsing, ensuring the stub file stays in perfect + sync with the implementation without any manual updates. + + Returns: + The complete content of the .pyi stub file as a string + """ + import ast + import inspect + import re + from pathlib import Path + + # Read and parse the source file + source_file = Path(__file__) + source_code = source_file.read_text() + tree = ast.parse(source_code) + + # Extract dataclasses + dataclass_defs = [] + dataclass_names = [] + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + # Check if it's a dataclass + has_dataclass_decorator = any( + (isinstance(d, ast.Name) and d.id == "dataclass") + or (isinstance(d, ast.Attribute) and d.attr == "dataclass") + for d in node.decorator_list + ) + if has_dataclass_decorator: + dataclass_names.append(node.name) + # Reconstruct the dataclass definition + fields = [] + for item in node.body: + if isinstance(item, ast.AnnAssign) and isinstance( + item.target, ast.Name + ): + field_name = item.target.id + # Get annotation as string + annotation = ast.unparse(item.annotation) + # Get default value if present + if item.value: + default = ast.unparse(item.value) + fields.append(f" {field_name}: {annotation} = {default}") + else: + fields.append(f" {field_name}: {annotation}") + + # Get docstring if present + docstring = "" + if ( + node.body + and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + ): + docstring = f' """{node.body[0].value.value}"""' + + dataclass_def = f"@dataclass\nclass {node.name}:\n" + if docstring: + dataclass_def += f"{docstring}\n" + dataclass_def += "\n".join(fields) if fields else " pass" + dataclass_defs.append(dataclass_def + "\n") + + # Get methods from KubernetesController using inspect + methods = [] + for name, method in inspect.getmembers( + KubernetesController, predicate=inspect.isfunction + ): + if name.startswith("_"): + continue + + sig = inspect.signature(method) + params = [] + seen_keyword_only_separator = False + + for param_name, param in sig.parameters.items(): + if param_name == "self": + continue + + # Add keyword-only separator if this is the first keyword-only param + if ( + param.kind == inspect.Parameter.KEYWORD_ONLY + and not seen_keyword_only_separator + ): + params.append("*") + seen_keyword_only_separator = True + + param_str = param_name + if param.annotation != inspect.Parameter.empty: + annotation = param.annotation + if isinstance(annotation, type): + annotation = annotation.__name__ + else: + annotation = str(annotation).replace("typing.", "") + param_str += f": {annotation}" + + if param.default != inspect.Parameter.empty: + if param.default is None: + param_str += " = None" + elif isinstance(param.default, str): + param_str += f' = "{param.default}"' + elif isinstance(param.default, bool): + param_str += f" = {param.default}" + else: + param_str += f" = {param.default}" + + params.append(param_str) + + params_str = ", ".join(params) + + return_annotation = sig.return_annotation + if return_annotation == inspect.Signature.empty: + return_type = "Any" + else: + return_type_str = str(return_annotation) + match = re.search(r"Coroutine\[Any, Any, (.+)\]", return_type_str) + if match: + return_type = match.group(1) + else: + return_type = return_type_str.replace("typing.", "") + + method_stub = f" def {name}(self, {params_str}) -> {return_type}: ..." + methods.append(method_stub) + + # Build __all__ export list + all_exports = dataclass_names + ["KubernetesController", "KubernetesControllerSync"] + + # Build async method stubs (for KubernetesController abstract class) + async_methods = [] + for name, method in inspect.getmembers( + KubernetesController, predicate=inspect.isfunction + ): + if name.startswith("_"): + continue + + sig = inspect.signature(method) + params = [] + seen_keyword_only_separator = False + + for param_name, param in sig.parameters.items(): + if param_name == "self": + continue + + # Add keyword-only separator if this is the first keyword-only param + if ( + param.kind == inspect.Parameter.KEYWORD_ONLY + and not seen_keyword_only_separator + ): + params.append("*") + seen_keyword_only_separator = True + + param_str = param_name + if param.annotation != inspect.Parameter.empty: + annotation = param.annotation + if isinstance(annotation, type): + annotation = annotation.__name__ + else: + annotation = str(annotation).replace("typing.", "") + param_str += f": {annotation}" + + if param.default != inspect.Parameter.empty: + if param.default is None: + param_str += " = None" + elif isinstance(param.default, str): + param_str += f' = "{param.default}"' + elif isinstance(param.default, bool): + param_str += f" = {param.default}" + else: + param_str += f" = {param.default}" + + params.append(param_str) + + params_str = ", ".join(params) + + return_annotation = sig.return_annotation + if return_annotation == inspect.Signature.empty: + return_type = "Any" + else: + return_type_str = str(return_annotation) + # Keep the Coroutine wrapper for async methods + return_type = return_type_str.replace("typing.", "") + + async_method_stub = ( + f" async def {name}(self, {params_str}) -> {return_type}: ..." + ) + async_methods.append(async_method_stub) + + # Build the complete stub file content + stub_content = f'''"""Type stubs for controller module. + +This file is AUTO-GENERATED by running: + python -m src.infra.k8s.controller + +Do not edit manually. Regenerate after updating KubernetesController. +Dataclasses are automatically extracted from source using AST parsing. +""" + +from __future__ import annotations + +from abc import ABC +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +__all__ = {all_exports!r} + + +# ============================================================================= +# Data Types (AUTO-EXTRACTED) +# ============================================================================= + +{chr(10).join(dataclass_defs)} + + +# ============================================================================= +# Abstract Controller +# ============================================================================= + +class KubernetesController(ABC): + """Abstract base class for Kubernetes operations. + + All methods are async to support both sync (kubectl) and async (kr8s) + implementations. Use `run_sync()` to call from synchronous code. + """ + +{chr(10).join(async_methods)} + + +# ============================================================================= +# Synchronous Wrapper +# ============================================================================= + +class KubernetesControllerSync: + """Synchronous wrapper for KubernetesController with full type hints. + + All async methods from KubernetesController are exposed as synchronous methods. + The underlying async controller is wrapped automatically using run_sync(). + """ + + def __init__(self, controller: KubernetesController) -> None: + """Initialize the synchronous wrapper. + + Args: + controller: The underlying async KubernetesController instance + """ + ... + +{chr(10).join(methods)} +''' + + return stub_content + + +if __name__ == "__main__": + """Generate KubernetesControllerSync stub file.""" + from pathlib import Path + + # Generate stub content + stub_content = generate_sync_stubs() + + # Write to .pyi file next to this module + stub_path = Path(__file__).with_suffix(".pyi") + stub_path.write_text(stub_content) + + print(f"✅ Generated type stubs: {stub_path}") + print(f"📝 {len(stub_content.splitlines())} lines") + print("\nTo use the synchronous wrapper with full type hints:") + print(" from src.infra.k8s.controller import KubernetesControllerSync") + print(" from src.infra.k8s.kubectl import KubectlController") + print("") + print(" sync_controller = KubernetesControllerSync(KubectlController())") + print(" pods = sync_controller.get_pods('my-namespace') # Fully typed!") diff --git a/src/infra/k8s/controller.pyi b/src/infra/k8s/controller.pyi new file mode 100644 index 0000000..3bfa632 --- /dev/null +++ b/src/infra/k8s/controller.pyi @@ -0,0 +1,288 @@ +"""Type stubs for controller module. + +This file is AUTO-GENERATED by running: + python -m src.infra.k8s.controller + +Do not edit manually. Regenerate after updating KubernetesController. +Dataclasses are automatically extracted from source using AST parsing. +""" + +from __future__ import annotations + +from abc import ABC +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +__all__ = [ + "CommandResult", + "PodInfo", + "ReplicaSetInfo", + "JobInfo", + "ServiceInfo", + "ClusterIssuerStatus", + "KubernetesController", + "KubernetesControllerSync", +] + +# ============================================================================= +# Data Types (AUTO-EXTRACTED) +# ============================================================================= + +@dataclass +class CommandResult: + """Result of a command execution.""" + + success: bool + stdout: str = "" + stderr: str = "" + returncode: int = 0 + +@dataclass +class PodInfo: + """Information about a Kubernetes pod.""" + + name: str + status: str + restarts: int = 0 + creation_timestamp: str = "" + job_owner: str = "" + ip: str = "" + node: str = "" + +@dataclass +class ReplicaSetInfo: + """Information about a Kubernetes ReplicaSet.""" + + name: str + replicas: int + revision: str = "" + created_at: datetime | None = None + owner_deployment: str | None = None + +@dataclass +class JobInfo: + """Information about a Kubernetes Job.""" + + name: str + status: str + +@dataclass +class ServiceInfo: + """Information about a Kubernetes Service.""" + + name: str + type: str + cluster_ip: str + external_ip: str = "" + ports: str = "" + +@dataclass +class ClusterIssuerStatus: + """Status of a cert-manager ClusterIssuer.""" + + exists: bool + ready: bool + message: str = "" + +# ============================================================================= +# Abstract Controller +# ============================================================================= + +class KubernetesController(ABC): + """Abstract base class for Kubernetes operations. + + All methods are async to support both sync (kubectl) and async (kr8s) + implementations. Use `run_sync()` to call from synchronous code. + """ + + async def apply_manifest(self, manifest_path: Path) -> CommandResult: ... + async def check_cert_manager_installed( + self, + ) -> bool: ... + async def delete_helm_secrets( + self, namespace: str, release_name: str + ) -> CommandResult: ... + async def delete_namespace( + self, namespace: str, *, wait: bool = True, timeout: str = "120s" + ) -> CommandResult: ... + async def delete_pvcs(self, namespace: str) -> CommandResult: ... + async def delete_replicaset(self, name: str, namespace: str) -> CommandResult: ... + async def delete_resource( + self, + resource_type: str, + name: str, + namespace: str, + *, + cascade: str | None = None, + wait: bool = True, + ) -> CommandResult: ... + async def delete_resources_by_label( + self, + resource_types: str, + namespace: str, + label_selector: str, + *, + force: bool = False, + cascade: str | None = None, + ) -> CommandResult: ... + async def get_cluster_issuer_status( + self, issuer_name: str + ) -> ClusterIssuerStatus: ... + async def get_cluster_issuer_yaml(self, issuer_name: str) -> str | None: ... + async def get_current_context( + self, + ) -> str: ... + async def get_deployment_revision( + self, name: str, namespace: str + ) -> str | None: ... + async def get_deployments(self, namespace: str) -> list[str]: ... + async def get_jobs(self, namespace: str) -> list[JobInfo]: ... + async def get_pod_logs( + self, + namespace: str, + pod: str | None = None, + *, + container: str | None = None, + label_selector: str | None = None, + follow: bool = False, + tail: int = 100, + previous: bool = False, + ) -> CommandResult: ... + async def get_pods( + self, namespace: str, label_selector: str | None = None + ) -> list[PodInfo]: ... + async def get_replicasets(self, namespace: str) -> list[ReplicaSetInfo]: ... + async def get_services(self, namespace: str) -> list[ServiceInfo]: ... + async def is_minikube_context( + self, + ) -> bool: ... + async def namespace_exists(self, namespace: str) -> bool: ... + async def resource_exists( + self, resource_type: str, name: str, namespace: str + ) -> bool: ... + async def rollout_restart( + self, resource_type: str, namespace: str, name: str | None = None + ) -> CommandResult: ... + async def rollout_status( + self, + resource_type: str, + namespace: str, + name: str | None = None, + *, + timeout: str = "300s", + ) -> CommandResult: ... + async def scale_replicaset( + self, name: str, namespace: str, replicas: int + ) -> CommandResult: ... + async def wait_for_pods( + self, + namespace: str, + label_selector: str, + *, + condition: str = "ready", + timeout: str = "300s", + ) -> CommandResult: ... + +# ============================================================================= +# Synchronous Wrapper +# ============================================================================= + +class KubernetesControllerSync: + """Synchronous wrapper for KubernetesController with full type hints. + + All async methods from KubernetesController are exposed as synchronous methods. + The underlying async controller is wrapped automatically using run_sync(). + """ + + def __init__(self, controller: KubernetesController) -> None: + """Initialize the synchronous wrapper. + + Args: + controller: The underlying async KubernetesController instance + """ + ... + + def apply_manifest(self, manifest_path: Path) -> CommandResult: ... + def check_cert_manager_installed( + self, + ) -> bool: ... + def delete_helm_secrets( + self, namespace: str, release_name: str + ) -> CommandResult: ... + def delete_namespace( + self, namespace: str, *, wait: bool = True, timeout: str = "120s" + ) -> CommandResult: ... + def delete_pvcs(self, namespace: str) -> CommandResult: ... + def delete_replicaset(self, name: str, namespace: str) -> CommandResult: ... + def delete_resource( + self, + resource_type: str, + name: str, + namespace: str, + *, + cascade: str | None = None, + wait: bool = True, + ) -> CommandResult: ... + def delete_resources_by_label( + self, + resource_types: str, + namespace: str, + label_selector: str, + *, + force: bool = False, + cascade: str | None = None, + ) -> CommandResult: ... + def get_cluster_issuer_status(self, issuer_name: str) -> ClusterIssuerStatus: ... + def get_cluster_issuer_yaml(self, issuer_name: str) -> str | None: ... + def get_current_context( + self, + ) -> str: ... + def get_deployment_revision(self, name: str, namespace: str) -> str | None: ... + def get_deployments(self, namespace: str) -> list[str]: ... + def get_jobs(self, namespace: str) -> list[JobInfo]: ... + def get_pod_logs( + self, + namespace: str, + pod: str | None = None, + *, + container: str | None = None, + label_selector: str | None = None, + follow: bool = False, + tail: int = 100, + previous: bool = False, + ) -> CommandResult: ... + def get_pods( + self, namespace: str, label_selector: str | None = None + ) -> list[PodInfo]: ... + def get_replicasets(self, namespace: str) -> list[ReplicaSetInfo]: ... + def get_services(self, namespace: str) -> list[ServiceInfo]: ... + def is_minikube_context( + self, + ) -> bool: ... + def namespace_exists(self, namespace: str) -> bool: ... + def resource_exists( + self, resource_type: str, name: str, namespace: str + ) -> bool: ... + def rollout_restart( + self, resource_type: str, namespace: str, name: str | None = None + ) -> CommandResult: ... + def rollout_status( + self, + resource_type: str, + namespace: str, + name: str | None = None, + *, + timeout: str = "300s", + ) -> CommandResult: ... + def scale_replicaset( + self, name: str, namespace: str, replicas: int + ) -> CommandResult: ... + def wait_for_pods( + self, + namespace: str, + label_selector: str, + *, + condition: str = "ready", + timeout: str = "300s", + ) -> CommandResult: ... diff --git a/src/infra/k8s/helpers.py b/src/infra/k8s/helpers.py new file mode 100644 index 0000000..d8b4526 --- /dev/null +++ b/src/infra/k8s/helpers.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import os + +from cachetools.func import lru_cache # type: ignore + +from src.infra.constants import DEFAULT_CONSTANTS +from src.infra.k8s.controller import KubernetesController, KubernetesControllerSync + + +@lru_cache(maxsize=1) +def get_k8s_controller() -> KubernetesController: + """Get an instance of the KubernetesController. + + Returns: + An instance of KubernetesController + """ + from src.infra.k8s.kr8s_controller import Kr8sController + + return Kr8sController() + + +@lru_cache(maxsize=1) +def get_k8s_controller_sync() -> KubernetesControllerSync: + """Get a synchronous wrapper for KubernetesController. + + Returns: + An instance of KubernetesControllerSync wrapping the async controller + """ + from src.infra.k8s.kr8s_controller import Kr8sController + + controller = Kr8sController() + return KubernetesControllerSync(controller) + + +def get_namespace() -> str: + """Get the Kubernetes namespace from config or default.""" + return os.environ.get("K8S_NAMESPACE", DEFAULT_CONSTANTS.DEFAULT_NAMESPACE) + + +def get_postgres_label() -> str: + """Get the PostgreSQL pod label selector.""" + return DEFAULT_CONSTANTS.POSTGRES_POD_LABEL diff --git a/src/infra/k8s/kr8s_controller.py b/src/infra/k8s/kr8s_controller.py index 5c0ff37..fb45a92 100644 --- a/src/infra/k8s/kr8s_controller.py +++ b/src/infra/k8s/kr8s_controller.py @@ -181,6 +181,55 @@ def _run() -> CommandResult: return await asyncio.to_thread(_run) + async def resource_exists( + self, + resource_type: str, + name: str, + namespace: str, + ) -> bool: + """Check if a Kubernetes resource exists.""" + import subprocess + + def _run() -> bool: + result = subprocess.run( + ["kubectl", "get", resource_type, name, "-n", namespace], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + return await asyncio.to_thread(_run) + + async def delete_resource( + self, + resource_type: str, + name: str, + namespace: str, + *, + cascade: str | None = None, + wait: bool = True, + ) -> CommandResult: + """Delete a specific Kubernetes resource by name.""" + import subprocess + + def _run() -> CommandResult: + cmd = ["kubectl", "delete", resource_type, name, "-n", namespace] + if cascade: + cmd.append(f"--cascade={cascade}") + if wait: + cmd.append("--wait=true") + else: + cmd.append("--wait=false") + result = subprocess.run(cmd, capture_output=True, text=True) + return CommandResult( + success=result.returncode == 0, + stdout=result.stdout or "", + stderr=result.stderr or "", + returncode=result.returncode, + ) + + return await asyncio.to_thread(_run) + async def delete_resources_by_label( self, resource_types: str, @@ -188,6 +237,7 @@ async def delete_resources_by_label( label_selector: str, *, force: bool = False, + cascade: str | None = None, ) -> CommandResult: """Delete Kubernetes resources matching a label selector. @@ -206,6 +256,8 @@ async def delete_resources_by_label( ] if force: cmd.extend(["--force", "--grace-period=0"]) + if cascade: + cmd.append(f"--cascade={cascade}") def _run() -> CommandResult: result = subprocess.run(cmd, capture_output=True, text=True) @@ -440,13 +492,30 @@ async def scale_replicaset( # Pod Operations # ========================================================================= - async def get_pods(self, namespace: str) -> list[PodInfo]: - """Get all pods in a namespace with their status.""" + async def get_pods( + self, + namespace: str, + label_selector: str | None = None, + ) -> list[PodInfo]: + """Get all pods in a namespace with their status. + + Args: + namespace: Kubernetes namespace to search + label_selector: Optional label selector to filter pods (e.g., "app=postgres") + + Returns: + List of PodInfo objects matching the criteria + """ try: api = await self._get_api() result = [] - async for pod in Pod.list(namespace=namespace, api=api): + # kr8s Pod.list accepts label_selector as a dict or string + list_kwargs = {"namespace": namespace, "api": api} + if label_selector: + list_kwargs["label_selector"] = label_selector + + async for pod in Pod.list(**list_kwargs): metadata = pod.metadata spec = pod.spec status = pod.status diff --git a/src/infra/k8s/kubectl_controller.py b/src/infra/k8s/kubectl_controller.py index 376e366..979bce6 100644 --- a/src/infra/k8s/kubectl_controller.py +++ b/src/infra/k8s/kubectl_controller.py @@ -116,6 +116,35 @@ async def apply_manifest(self, manifest_path: Path) -> CommandResult: """Apply a Kubernetes manifest file.""" return await self._run_kubectl(["apply", "-f", str(manifest_path)]) + async def resource_exists( + self, + resource_type: str, + name: str, + namespace: str, + ) -> bool: + """Check if a Kubernetes resource exists.""" + result = await self._run_kubectl(["get", resource_type, name, "-n", namespace]) + return result.success + + async def delete_resource( + self, + resource_type: str, + name: str, + namespace: str, + *, + cascade: str | None = None, + wait: bool = True, + ) -> CommandResult: + """Delete a specific Kubernetes resource by name.""" + args = ["delete", resource_type, name, "-n", namespace] + if cascade: + args.append(f"--cascade={cascade}") + if wait: + args.append("--wait=true") + else: + args.append("--wait=false") + return await self._run_kubectl(args) + async def delete_resources_by_label( self, resource_types: str, @@ -123,6 +152,7 @@ async def delete_resources_by_label( label_selector: str, *, force: bool = False, + cascade: str | None = None, ) -> CommandResult: """Delete Kubernetes resources matching a label selector.""" args = [ @@ -135,6 +165,8 @@ async def delete_resources_by_label( ] if force: args.extend(["--force", "--grace-period=0"]) + if cascade: + args.append(f"--cascade={cascade}") return await self._run_kubectl(args) async def delete_helm_secrets( @@ -326,9 +358,25 @@ async def scale_replicaset( # Pod Operations # ========================================================================= - async def get_pods(self, namespace: str) -> list[PodInfo]: - """Get all pods in a namespace with their status.""" - result = await self._run_kubectl(["get", "pods", "-n", namespace, "-o", "json"]) + async def get_pods( + self, + namespace: str, + label_selector: str | None = None, + ) -> list[PodInfo]: + """Get all pods in a namespace with their status. + + Args: + namespace: Kubernetes namespace to search + label_selector: Optional label selector to filter pods (e.g., "app=postgres") + + Returns: + List of PodInfo objects matching the criteria + """ + args = ["get", "pods", "-n", namespace, "-o", "json"] + if label_selector: + args.extend(["-l", label_selector]) + + result = await self._run_kubectl(args) if not result.success or not result.stdout: return [] diff --git a/src/infra/k8s/port_forward.py b/src/infra/k8s/port_forward.py new file mode 100644 index 0000000..aa33cfc --- /dev/null +++ b/src/infra/k8s/port_forward.py @@ -0,0 +1,478 @@ +"""PostgreSQL port forwarding context manager for Kubernetes. + +Provides automatic port forwarding to PostgreSQL pods in Kubernetes +for CLI operations. +""" + +import socket +import subprocess +import time +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from functools import lru_cache, wraps +from typing import Any, TypeVar + +from rich.progress import Console + +from src.infra.constants import DEFAULT_CONSTANTS +from src.infra.k8s.helpers import ( + get_k8s_controller, + get_namespace, + get_postgres_label, +) +from src.infra.k8s.utils import run_sync +from src.infra.utils.service_config import is_bundled_postgres_enabled + +CONTROLLER = get_k8s_controller() + +# Type variable for function return type +T = TypeVar("T") + + +class PortForwardError(Exception): + """Error during port forwarding setup.""" + + +@dataclass +class PortForwardKey: + """Key for tracking active port forwards.""" + + namespace: str + pod_name: str + local_port: int + remote_port: int + + def __hash__(self) -> int: + return hash((self.namespace, self.pod_name, self.local_port, self.remote_port)) + + +@dataclass +class PortForwardProcess: + """Tracks an active port forward process.""" + + process: subprocess.Popen[str] + ref_count: int = 1 + + +# Global registry of active port forwards +_active_forwards: dict[PortForwardKey, PortForwardProcess] = {} + + +def _cleanup_stale_forwards() -> None: + """Remove any dead port-forward processes from the registry. + + This handles cases where the port-forward process died unexpectedly + (e.g., pod restart) but wasn't properly cleaned up. + """ + stale_keys = [] + for key, forward in _active_forwards.items(): + if forward.process.poll() is not None: + # Process has terminated + stale_keys.append(key) + + for key in stale_keys: + del _active_forwards[key] + + +def _is_port_in_use(port: int, host: str = "localhost") -> bool: + """Check if a local port is already in use. + + Args: + port: Port number to check + host: Host to check on (default: localhost) + + Returns: + True if port is in use, False otherwise + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.bind((host, port)) + return False + except OSError: + return True + + +@lru_cache(maxsize=1) +def _get_postgres_pod() -> str | None: + """Get the name of the PostgreSQL pod.""" + p_pods = run_sync( + CONTROLLER.get_pods(get_namespace(), label_selector=get_postgres_label()) + ) + if p_pods: + # Type narrowing: if list is non-empty, first element exists + pod = p_pods[0] + return pod.name + + return None + + +@contextmanager +def postgres_port_forward( + namespace: str, + console: Console | None = None, + *, + pod_name: str | None = None, + pod_label: str | None = None, + local_port: int = DEFAULT_CONSTANTS.DEFAULT_EPHEMERAL_PORT, + remote_port: int = 5432, + wait_time: float = 2.0, + reuse_existing: bool = True, +) -> Generator[None]: + """Context manager for PostgreSQL port forwarding. + + Automatically sets up and tears down kubectl port-forward for PostgreSQL access. + Uses reference counting to allow nested/concurrent calls to reuse the same + port-forward process. + + Args: + namespace: Kubernetes namespace containing the pod + console: Rich console for output + pod_name: Name of the PostgreSQL pod + pod_label: Label selector to find pod (if pod_name not provided) + local_port: Local port to forward to (default: 5432) + remote_port: Remote port on the pod (default: 5432) + wait_time: Time to wait for port-forward to be ready (default: 2.0s) + reuse_existing: If True, reuse existing forward if available (default: True) + + Yields: + None - port forwarding is active during context + + Raises: + PortForwardError: If port forwarding fails to start or port is in use + + Example: + >>> with postgres_port_forward("api-forge-prod", pod_name="postgres-0"): + ... # Port 5432 is now forwarded to postgres-0:5432 + ... conn = psycopg2.connect(host="localhost", port=5432, ...) + ... # Do database operations + ... # Port forwarding automatically stopped (when last reference exits) + + >>> # Nested calls reuse the same forward: + >>> with postgres_port_forward("api-forge-prod", pod_name="postgres-0"): + ... # First forward starts + ... with postgres_port_forward("api-forge-prod", pod_name="postgres-0"): + ... # Reuses existing forward (ref_count=2) + ... pass + ... # ref_count=1, forward still active + ... # ref_count=0, forward stopped + """ + if not pod_name: + if not pod_label: + raise PortForwardError("Either pod_name or pod_label must be provided") + pod_name = _get_postgres_pod() + if not pod_name: + raise PortForwardError( + f"No pod found with label '{pod_label}' in namespace '{namespace}'" + ) + + # Create key for this forward + key = PortForwardKey( + namespace=namespace, + pod_name=pod_name, + local_port=local_port, + remote_port=remote_port, + ) + + # Check if we already have an active forward for this config + forward: PortForwardProcess | None = None + if reuse_existing and key in _active_forwards: + forward = _active_forwards[key] + # Verify process is still alive + if forward.process.poll() is not None: + # Process died, remove stale entry and create new + del _active_forwards[key] + forward = None + + # Create new forward if needed + if forward is None: + # Check if port is already in use by something else + if _is_port_in_use(local_port): + # Before failing, cleanup any stale entries that might be lingering + _cleanup_stale_forwards() + # Check again after cleanup + if _is_port_in_use(local_port): + raise PortForwardError( + f"Port {local_port} is already in use and no existing forward found." + ) + + cmd = [ + "kubectl", + "port-forward", + "-n", + namespace, + pod_name, + f"{local_port}:{remote_port}", + ] + + if console: + console.print( + f"[dim]Starting port-forward: {pod_name} {local_port}:{remote_port}[/dim]" + ) + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + # Wait for port-forward to be ready + time.sleep(wait_time) + + # Check if process started successfully + if process.poll() is not None: + _, stderr = process.communicate() + raise PortForwardError(f"Port forward failed to start: {stderr.strip()}") + + forward = PortForwardProcess(process=process, ref_count=0) + _active_forwards[key] = forward + + if console: + console.print( + f"[dim]Port-forward active: localhost:{local_port} -> {pod_name}:{remote_port}[/dim]" + ) + else: + if console: + console.print( + f"[dim]Reusing existing port-forward: {pod_name} " + f"{local_port}:{remote_port}[/dim]" + ) + + # Increment ref count + forward.ref_count += 1 + + try: + yield + finally: + # Decrement ref count + forward.ref_count -= 1 + + if console: + console.print( + f"[dim]Released port-forward reference (refs={forward.ref_count})[/dim]" + ) + + # Clean up if last reference + if forward.ref_count == 0: + if forward.process.poll() is None: + if console: + console.print("[dim]Stopping port-forward...[/dim]") + forward.process.terminate() + try: + forward.process.wait(timeout=5) + except subprocess.TimeoutExpired: + forward.process.kill() + forward.process.wait() + + # Remove from registry (check identity in case it was replaced) + if key in _active_forwards and _active_forwards[key] is forward: + del _active_forwards[key] + + if console: + console.print("[dim]Port-forward stopped[/dim]") + + +def with_postgres_port_forward( + namespace: str | None = None, + *, + pod_name: str | None = None, + pod_label: str | None = None, + local_port: int = DEFAULT_CONSTANTS.DEFAULT_EPHEMERAL_PORT, + remote_port: int = 5432, + wait_time: float = 5.0, +) -> Callable[[Callable[..., T]], Callable[..., T]]: + """Decorator to automatically set up port forwarding for a function. + + This decorator wraps a function with the postgres_port_forward context manager, + automatically handling port forwarding setup and teardown. + + Args: + namespace: Kubernetes namespace (if None, tries to get from function kwargs) + pod_name: Name of the PostgreSQL pod (if None, tries to get from function kwargs) + local_port: Local port to forward to (default: 5432) + remote_port: Remote port on the pod (default: 5432) + wait_time: Time to wait for port-forward to be ready (default: 2.0s) + + Returns: + Decorator function + + Example: + >>> @with_postgres_port_forward(namespace="api-forge-prod", pod_name="postgres-0") + ... def initialize_database(): + ... # Port forwarding is active here + ... conn = psycopg2.connect(host="localhost", port=5432, ...) + ... # Do database operations + ... + >>> # Or let it extract from function arguments: + >>> @with_postgres_port_forward() + ... def verify_db(namespace: str, pod: str): + ... # Decorator will use namespace and pod arguments + ... conn = psycopg2.connect(host="localhost", port=5432, ...) + """ + + def decorator(func: Callable[..., T]) -> Callable[..., T]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> T: + # Try to extract namespace and pod_name from function arguments + actual_namespace = namespace + actual_pod_name = pod_name + actual_pod_label = pod_label + + # If not provided to decorator, try to get from kwargs + if actual_namespace is None: + actual_namespace = kwargs.get("namespace") + if actual_pod_name is None: + actual_pod_name = kwargs.get("pod_name") or kwargs.get("pod") + if actual_pod_label is None: + actual_pod_label = kwargs.get("pod_label") + + # Validate we have required parameters + if not actual_namespace or (not actual_pod_name and not actual_pod_label): + raise ValueError( + "namespace and pod_name or pod_label must be provided either to decorator " + "or as function arguments" + ) + + # Execute function within port-forward context + with postgres_port_forward( + namespace=actual_namespace, + pod_name=actual_pod_name, + pod_label=actual_pod_label, + local_port=local_port, + remote_port=remote_port, + wait_time=wait_time, + ): + return func(*args, **kwargs) + + return wrapper + + return decorator + + +def with_postgres_port_forward_if_needed( + namespace: str | None = None, + *, + pod_name: str | None = None, + pod_label: str | None = None, + local_port: int = DEFAULT_CONSTANTS.DEFAULT_EPHEMERAL_PORT, + remote_port: int = 5432, + wait_time: float = 5.0, +) -> Callable[[Callable[..., T]], Callable[..., T]]: + """Decorator that sets up port forwarding only if bundled postgres is enabled. + + When bundled postgres is disabled (using external database), this is a no-op. + When bundled postgres is enabled, this wraps the function with port forwarding. + + Args: + namespace: Kubernetes namespace (if None, tries to get from function kwargs) + pod_name: Name of the PostgreSQL pod (if None, tries to get from function kwargs) + pod_label: Label selector to find pod (if pod_name not provided) + local_port: Local port to forward to (default: 5432) + remote_port: Remote port on the pod (default: 5432) + wait_time: Time to wait for port-forward to be ready (default: 5.0s) + + Returns: + Decorator function + + Example: + >>> @with_postgres_port_forward_if_needed(namespace="api-forge-prod") + ... def verify_database(): + ... # If bundled postgres: port forwarding is active + ... # If external postgres: no port forwarding needed + ... conn = psycopg2.connect(...) + """ + + def decorator(func: Callable[..., T]) -> Callable[..., T]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> T: + actual_namespace = namespace + actual_pod_name = pod_name + actual_pod_label = pod_label + + if actual_namespace is None: + actual_namespace = kwargs.get("namespace") + if actual_pod_name is None: + actual_pod_name = kwargs.get("pod_name") or kwargs.get("pod") + if actual_pod_label is None: + actual_pod_label = kwargs.get("pod_label") + + # Note: postgres_port_forward_if_needed handles the bundled check, + # but we still need namespace validation when bundled is enabled + if is_bundled_postgres_enabled(): + if not actual_namespace or ( + not actual_pod_name and not actual_pod_label + ): + raise ValueError( + "namespace and pod_name or pod_label must be provided either to decorator " + "or as function arguments" + ) + + with postgres_port_forward_if_needed( + namespace=actual_namespace or "", + pod_name=actual_pod_name, + pod_label=actual_pod_label, + local_port=local_port, + remote_port=remote_port, + wait_time=wait_time, + ): + return func(*args, **kwargs) + + return wrapper + + return decorator + + +@contextmanager +def postgres_port_forward_if_needed( + namespace: str, + console: Console | None = None, + *, + pod_name: str | None = None, + pod_label: str | None = None, + local_port: int = DEFAULT_CONSTANTS.DEFAULT_EPHEMERAL_PORT, + remote_port: int = 5432, + wait_time: float = 2.0, + reuse_existing: bool = True, +) -> Generator[None]: + """Context manager that sets up port forwarding only if bundled postgres is enabled. + + When bundled postgres is disabled (using external database), this is a no-op. + When bundled postgres is enabled, this sets up kubectl port-forward. + + Args: + namespace: Kubernetes namespace containing the pod + console: Rich console for output + pod_name: Name of the PostgreSQL pod + pod_label: Label selector to find pod (if pod_name not provided) + local_port: Local port to forward to (default: 5432) + remote_port: Remote port on the pod (default: 5432) + wait_time: Time to wait for port-forward to be ready (default: 2.0s) + reuse_existing: If True, reuse existing forward if available (default: True) + + Yields: + None + + Example: + >>> with postgres_port_forward_if_needed("api-forge-prod", pod_name="postgres-0"): + ... # If bundled postgres: port forwarding is active + ... # If external postgres: no port forwarding, direct connection + ... conn = psycopg2.connect(...) + """ + if not is_bundled_postgres_enabled(): + # No port forwarding needed for external postgres + yield + return + + # Use the regular port forward context manager + with postgres_port_forward( + namespace=namespace, + console=console, + pod_name=pod_name, + pod_label=pod_label, + local_port=local_port, + remote_port=remote_port, + wait_time=wait_time, + reuse_existing=reuse_existing, + ): + yield diff --git a/src/infra/k8s/postgres_connection.py b/src/infra/k8s/postgres_connection.py new file mode 100644 index 0000000..5fff920 --- /dev/null +++ b/src/infra/k8s/postgres_connection.py @@ -0,0 +1,116 @@ +from typing import Any, override + +from src.infra.constants import DEFAULT_CONSTANTS +from src.infra.k8s.helpers import get_namespace, get_postgres_label +from src.infra.k8s.port_forward import with_postgres_port_forward +from src.infra.postgres.connection import DbSettings, PostgresConnection +from src.infra.utils.service_config import is_bundled_postgres_enabled + +K8S_NAMESPACE = get_namespace() +POSTGRES_LABEL = get_postgres_label() + + +class K8sPostgresConnectionBundled(PostgresConnection): + def __init__( + self, + settings: DbSettings, + superuser_mode: bool = False, + ) -> None: + """PostgreSQL connection manager for bundled Postgres. + Args: + settings: Database settings + user: Override user (default: settings.user) + password: Override password (default: settings.password) + """ + super().__init__( + settings, + superuser_mode=superuser_mode, + ssl_mode="disable", + ) + + @override + def get_dsn(self, database: str | None = None) -> dict[str, Any]: + """Get connection parameters for psycopg2.connect(). + + Args: + database: Override database name + + Returns: + Dict of connection parameters + """ + return { + "host": "localhost", + "port": DEFAULT_CONSTANTS.DEFAULT_EPHEMERAL_PORT, + "dbname": database or self._settings.app_db, + "user": self._superuser_mode + and self._settings.superuser + or self._settings.user, + "password": self._superuser_mode + and self._settings.superuser_password + or self._settings.password, + "sslmode": self._ssl_mode, + } + + @override + @with_postgres_port_forward(namespace=K8S_NAMESPACE, pod_label=POSTGRES_LABEL) + def test_connection(self, database: str | None = None) -> tuple[bool, str]: + return super().test_connection(database) + + @override + @with_postgres_port_forward(namespace=K8S_NAMESPACE, pod_label=POSTGRES_LABEL) + def connect(self, database: str | None = None) -> Any: + return super().connect(database) + + @override + @with_postgres_port_forward(namespace=K8S_NAMESPACE, pod_label=POSTGRES_LABEL) + def execute( + self, + sql: str, + params: tuple[Any, ...] | None = None, + database: str | None = None, + ) -> list[dict[str, Any]]: + return super().execute(sql, params, database) + + @override + @with_postgres_port_forward(namespace=K8S_NAMESPACE, pod_label=POSTGRES_LABEL) + def execute_script(self, sql: str, database: str | None = None) -> None: + return super().execute_script(sql, database) + + @override + @with_postgres_port_forward(namespace=K8S_NAMESPACE, pod_label=POSTGRES_LABEL) + def scalar( + self, + sql: str, + params: tuple[Any, ...] | None = None, + database: str | None = None, + ) -> Any: + return super().scalar(sql, params, database) + + +class K8sPostgresConnection(PostgresConnection): ... + + +def get_k8s_postgres_connection( + settings: DbSettings, + superuser_mode: bool = False, +) -> PostgresConnection: + """Get a Kubernetes PostgreSQL connection. + + Args: + settings: Database settings + superuser_mode: Whether to connect as superuser + bundled_postgres: Whether to use bundled Postgres (port-forward) + + Returns: + PostgreSQL connection + """ + if is_bundled_postgres_enabled(): + return K8sPostgresConnectionBundled( + settings=settings, + superuser_mode=superuser_mode, + ) + else: + return K8sPostgresConnection( + settings=settings, + superuser_mode=superuser_mode, + ) diff --git a/src/infra/k8s/utils.py b/src/infra/k8s/utils.py index a7b1960..8f41bf8 100644 --- a/src/infra/k8s/utils.py +++ b/src/infra/k8s/utils.py @@ -30,18 +30,15 @@ def run_sync[T](coro: Coroutine[Any, Any, T]) -> T: pods = run_sync(controller.get_pods("my-namespace")) """ try: - loop = asyncio.get_running_loop() + asyncio.get_running_loop() + # We're inside an async context with a running loop + # Create a new loop in a thread to avoid blocking + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor() as pool: + future = pool.submit(asyncio.run, coro) + return future.result() except RuntimeError: # No running loop, create a new one - return asyncio.run(coro) - else: - # We're inside an async context, use run_until_complete - # This handles nested async calls - if loop.is_running(): - # Create a new loop in a thread to avoid blocking - import concurrent.futures - - with concurrent.futures.ThreadPoolExecutor() as pool: - future = pool.submit(asyncio.run, coro) - return future.result() - return loop.run_until_complete(coro) + result: T = asyncio.run(coro) + return result diff --git a/src/infra/postgres/__init__.py b/src/infra/postgres/__init__.py new file mode 100644 index 0000000..21b174d --- /dev/null +++ b/src/infra/postgres/__init__.py @@ -0,0 +1,26 @@ +"""PostgreSQL database management infrastructure. + +This module provides Python implementations for PostgreSQL database operations +including connection management, initialization, verification, password sync, +backup, and reset functionality. + +These replace the bash scripts previously in infra/docker/prod/postgres/ and +are used by the CLI `db` commands. +""" + +from .backup import PostgresBackup +from .connection import DbSettings, PostgresConnection +from .init import PostgresInitializer +from .reset import PostgresReset +from .sync import PostgresPasswordSync +from .verify import PostgresVerifier + +__all__ = [ + "DbSettings", + "PostgresConnection", + "PostgresVerifier", + "PostgresInitializer", + "PostgresPasswordSync", + "PostgresBackup", + "PostgresReset", +] diff --git a/src/infra/postgres/backup.py b/src/infra/postgres/backup.py new file mode 100644 index 0000000..8c881cc --- /dev/null +++ b/src/infra/postgres/backup.py @@ -0,0 +1,199 @@ +"""PostgreSQL backup functionality. + +Provides database backup using pg_dump with compression and checksums. +""" + +import gzip +import hashlib +import os +import subprocess +import time +from datetime import datetime +from pathlib import Path + +from src.cli.shared.console import console +from src.infra.postgres.connection import PostgresConnection + + +class PostgresBackup: + """Creates PostgreSQL database backups. + + Supports both custom format (for pg_restore) and SQL format. + Includes compression and SHA256 checksums. + """ + + def __init__( + self, connection: PostgresConnection, backup_dir: Path, retention_days: int = 7 + ) -> None: + self._settings = connection.settings + self._connection = connection + self._console = console + self.backup_dir = backup_dir + self.retention_days = retention_days + + def create_backup( + self, + *, + user: str | None = None, + password: str | None = None, + database: str | None = None, + ) -> tuple[bool, str]: + """Create a database backup. + + Args: + password: Password for database user (defaults to ro_user password) + database: Database to backup (defaults to app_db) + user: User for backup (defaults to ro_user for read-only access) + + Returns: + Tuple of (success, backup_path or error message) + """ + s = self._settings + + database = database or s.app_db + user = user or s.ro_user + + if not password: + s.ensure_ro_user_password() + + password = password or s.ro_user_password + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_name = f"backup_{database}_{timestamp}" + + self._console.print("\n[bold]== Creating PostgreSQL Backup ==[/bold]") + self._console.info(f"Database: {database}") + self._console.info(f"User: {user}") + self._console.info(f"Host: {s.host}:{s.port}") + self._console.info(f"Backup dir: {self.backup_dir}") + + # Ensure backup directory exists + self.backup_dir.mkdir(parents=True, exist_ok=True) + + custom_path = self.backup_dir / f"{backup_name}.dump" + sql_path = self.backup_dir / f"{backup_name}.sql" + sql_gz_path = self.backup_dir / f"{backup_name}.sql.gz" + + try: + # Create custom format backup + self._console.info("Creating custom format backup...") + env = {"PGPASSWORD": password} + # Remove any None values from env + env_clean = { + k: v for k, v in {**os.environ, **env}.items() if v is not None + } + result = subprocess.run( + [ + "pg_dump", + "-h", + s.host, + "-p", + str(s.port), + "-U", + user, + "-Fc", + "-f", + str(custom_path), + database, + ], + env=env_clean, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + return False, f"pg_dump failed: {result.stderr}" + self._console.ok(f"Custom format: {custom_path.name}") + + env_clean = { + k: v for k, v in {**os.environ, **env}.items() if v is not None + } + result = subprocess.run( + [ + "pg_dump", + "-h", + s.host, + "-p", + str(s.port), + "-U", + user, + "-f", + str(sql_path), + database, + ], + env=env_clean, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + return False, f"pg_dump SQL failed: {result.stderr}" + + # Compress SQL backup + self._console.info("Compressing SQL backup...") + with open(sql_path, "rb") as f_in: + with gzip.open(sql_gz_path, "wb") as f_out: + f_out.writelines(f_in) + sql_path.unlink() # Remove uncompressed + self._console.ok(f"Compressed SQL: {sql_gz_path.name}") + + # Create checksums + self._console.info("Creating checksums...") + for path in [custom_path, sql_gz_path]: + checksum = self._sha256(path) + checksum_path = path.with_suffix(path.suffix + ".sha256") + checksum_path.write_text(f"{checksum} {path.name}\n") + self._console.ok("Checksums created") + + # Clean old backups + self._cleanup_old_backups() + + return True, str(custom_path) + + except Exception as e: + return False, str(e) + + def _sha256(self, path: Path) -> str: + """Calculate SHA256 checksum of file.""" + sha256 = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha256.update(chunk) + return sha256.hexdigest() + + def _cleanup_old_backups(self) -> None: + """Remove backups older than retention_days.""" + cutoff = time.time() - (self.retention_days * 86400) + removed = 0 + + for path in self.backup_dir.glob("backup_*"): + if path.stat().st_mtime < cutoff: + path.unlink() + removed += 1 + + if removed: + self._console.info(f"Cleaned up {removed} old backup file(s)") + + +def create_backup( + connection: PostgresConnection, + backup_dir: Path, + user: str | None = None, + password: str | None = None, + database: str | None = None, +) -> tuple[bool, str | None]: + """Create a database backup. + + Args: + password: Password for database user + database: Database to backup + user: User to connect as + settings: Optional DbSettings (loaded if not provided) + backup_dir: Optional backup directory + + Returns: + Tuple of (success, backup_path or error message) + """ + + backup = PostgresBackup(connection, backup_dir=backup_dir) + return backup.create_backup(user=user, password=password, database=database) diff --git a/src/infra/postgres/connection.py b/src/infra/postgres/connection.py new file mode 100644 index 0000000..1813711 --- /dev/null +++ b/src/infra/postgres/connection.py @@ -0,0 +1,467 @@ +"""PostgreSQL connection management. + +Provides centralized database settings and connection utilities. +""" + +import os +from collections.abc import Callable +from functools import lru_cache, wraps +from typing import Any, Literal, Self, TypeVar +from urllib.parse import quote_plus, urlencode + +import psycopg2 +import psycopg2.extensions +import psycopg2.extras +from pydantic import BaseModel + +from src.app.runtime.config.config_data import DatabaseConfig +from src.app.runtime.config.config_loader import load_config +from src.cli.shared.secrets import get_password +from src.infra.constants import DEFAULT_PATHS + +# Type variable for function return type +T = TypeVar("T") + + +class DbSettings(BaseModel): + """Centralized database settings for CLI operations. + + Extends DatabaseConfig fields with parsed connection details. + This is the single source of truth for all db CLI commands. + """ + + # From DatabaseConfig + url: str + superuser: str + superuser_password: str | None = None + app_db: str + postgres_db: str + user: str + password: str | None = None + owner_user: str + ro_user: str + ro_user_password: str | None = None + + # Temporal settings + temporal_user: str + temporal_password: str | None = None + temporal_owner: str + temporal_db: str = "temporal" + temporal_vis_db: str = "temporal_visibility" + + # Parsed from URL + host: str = "localhost" + port: int = 5432 + + def ensure_app_password(self) -> Self: + """Ensure application user password is set. + + Raises: + ValueError: If any required password is missing + """ + if not self.password: + self.password = get_password( + f"App user ({self.user}) password: ", + "POSTGRES_APP_USER_PW", + ) + + return self + + def ensure_ro_user_password(self) -> Self: + """Ensure read-only user password is set. + + Raises: + ValueError: If any required password is missing + """ + if not self.ro_user_password: + self.ro_user_password = get_password( + f"Read-only user ({self.ro_user}) password: ", + "POSTGRES_APP_RO_PW", + ) + return self + + def ensure_temporal_user_password(self) -> Self: + """Ensure temporal user password is set. + + Raises: + ValueError: If any required password is missing + """ + if not self.temporal_password: + self.temporal_password = get_password( + f"Temporal user ({self.temporal_user}) password: ", + "POSTGRES_TEMPORAL_PW", + ) + + return self + + def ensure_superuser_password(self) -> Self: + """Ensure superuser password is set. + + Raises: + ValueError: If superuser password is missing + """ + if not self.superuser_password: + self.superuser_password = get_password( + f"Postgres superuser ({self.superuser}) password: ", + "POSTGRES_PASSWORD", + ) + return self + + def ensure_all_passwords(self) -> Self: + """Ensure all required passwords are set. + + Raises: + ValueError: If any required password is missing + """ + self.ensure_superuser_password() + self.ensure_app_password() + self.ensure_ro_user_password() + self.ensure_temporal_user_password() + return self + + @classmethod + def load( + cls, + db_config: DatabaseConfig, + ) -> "DbSettings": + """Load settings from application config. + + Args: + environment_mode: development or production + superuser_password: Optional superuser password override + + Returns: + DbSettings populated from ConfigData.database + """ + + # Parse host/port from URL + host = db_config.host or "localhost" + port = db_config.port or 5432 + + return cls( + superuser=db_config.pg_superuser, + postgres_db=db_config.pg_db, + temporal_owner=db_config.temporal_owner, + temporal_user=db_config.temporal_user, + url=db_config.url, + app_db=db_config.app_db, + user=db_config.user, + owner_user=db_config.owner_user, + ro_user=db_config.ro_user, + password=db_config.password, + host=host, + port=port, + ) + + +class PostgresConnection: + """PostgreSQL connection manager. + + Uses psycopg2 for database operations. + """ + + def __init__( + self, + settings: DbSettings, + superuser_mode: bool = False, + ssl_mode: Literal["disable", "require"] = "require", + ): + """PostgreSQL connection manager. + + Args: + settings: Database settings + user: Override user (default: settings.user) + password: Override password (default: settings.password) + """ + self._settings = settings + self._superuser_mode = superuser_mode + # self._bundled_postgres = bundled_postgres + # Disable SSL for bundled postgres connections (port-forward scenario) + # Use 'require' for remote connections to enforce SSL/TLS + # self._ssl_mode = "disable" if self._bundled_postgres else "require" + self._ssl_mode = ssl_mode + self._conn: Any | None = None + self._current_database: str | None = None # Track connected database + + def get_dsn(self, database: str | None = None) -> dict[str, Any]: + """Get connection parameters for psycopg2.connect(). + + Args: + database: Override database name + + Returns: + Dict of connection parameters + """ + + if self._superuser_mode and not self._settings.superuser_password: + self._settings.ensure_superuser_password() + + return { + "host": self._settings.host, + "port": self._settings.port, + "dbname": database or self._settings.app_db, + "user": self._superuser_mode + and self._settings.superuser + or self._settings.user, + "password": self._superuser_mode + and self._settings.superuser_password + or self._settings.password + or "", + "sslmode": self._ssl_mode, + "connect_timeout": 5, + } + + def get_connection_string(self, database: str | None = None) -> str: + """Get a SQLAlchemy-compatible PostgreSQL connection string. + + This is primarily used for Alembic/SQLAlchemy subprocesses. + It is derived from `get_dsn()` so it works correctly for bundled, + external, docker-compose, and port-forwarded connections. + + Returns: + A URL like: postgresql://user:password@host:port/dbname?sslmode=require + """ + + dsn = self.get_dsn(database) + user = quote_plus(str(dsn.get("user", ""))) + password = quote_plus(str(dsn.get("password", ""))) + host = str(dsn.get("host", "localhost")) + port = str(dsn.get("port", "5432")) + dbname = str(dsn.get("dbname", "postgres")) + + query: dict[str, str] = {} + sslmode = dsn.get("sslmode") + if sslmode is not None: + query["sslmode"] = str(sslmode) + connect_timeout = dsn.get("connect_timeout") + if connect_timeout is not None: + query["connect_timeout"] = str(connect_timeout) + + query_str = urlencode(query) if query else "" + suffix = f"?{query_str}" if query_str else "" + + return f"postgresql://{user}:{password}@{host}:{port}/{dbname}{suffix}" + + def ensure_connected(self, database: str | None = None) -> Any: + """Ensure a connection exists, creating one if needed. + + Args: + database: Override database name + + Returns: + Active connection + """ + target_db = database or self._settings.app_db + # Reconnect if connection is closed or we need a different database + if ( + self._conn is None + or self._conn.closed + or self._current_database != target_db + ): + self.close() # Close existing connection if any + self._conn = psycopg2.connect(**self.get_dsn(database)) + self._current_database = target_db + return self._conn + + def connect(self, database: str | None = None) -> Any: + """Establish a connection to the database. + + Note: Connection is now automatically established on first use. + This method is provided for explicit connection control. + """ + self.close() # Close existing connection if any + target_db = database or self._settings.app_db + self._conn = psycopg2.connect(**self.get_dsn(database)) + self._current_database = target_db + return self._conn + + def close(self) -> None: + """Close the current connection if open.""" + if self._conn and not self._conn.closed: + self._conn.close() + self._conn = None + self._current_database = None + + def test_connection(self, database: str | None = None) -> tuple[bool, str]: + """Test database connectivity. + + Creates a separate test connection without affecting the main connection. + + Returns: + Tuple of (success, message) + """ + try: + with psycopg2.connect(**self.get_dsn(database)) as conn: + with conn.cursor() as cur: + cur.execute("SELECT version()") + row = cur.fetchone() + if row: + return True, f"Connected: {row[0]}" + return True, "Connected" + except psycopg2.OperationalError as e: + print(e) + return False, f"Connection failed: {e}" + except Exception as e: + print(e) + return False, f"Error: {e}" + + def execute( + self, + sql: str, + params: tuple[Any, ...] | None = None, + database: str | None = None, + ) -> list[dict[str, Any]]: + """Execute SQL and return results as list of dicts. + + Reuses the persistent connection. Use within a context manager or + call close() when done to properly cleanup. + """ + conn = self.ensure_connected(database) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(sql, params) + if cur.description: + return [dict(row) for row in cur.fetchall()] + conn.commit() + return [] + + def execute_script(self, sql: str, database: str | None = None) -> None: + """Execute a SQL script with autocommit. + + Reuses the persistent connection. Use within a context manager or + call close() when done to properly cleanup. + """ + conn = self.ensure_connected(database) + # Must commit/rollback any pending transaction before changing autocommit + if conn.status != psycopg2.extensions.STATUS_READY: + conn.rollback() + old_autocommit = conn.autocommit + try: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute(sql) + finally: + conn.autocommit = old_autocommit + + def scalar( + self, + sql: str, + params: tuple[Any, ...] | None = None, + database: str | None = None, + ) -> Any: + """Execute SQL and return single scalar value. + + Reuses the persistent connection. Use within a context manager or + call close() when done to properly cleanup. + """ + result = self.execute(sql, params, database) + if result and result[0]: + return list(result[0].values())[0] + return None + + @property + def settings(self) -> DbSettings: + """Get the database settings.""" + return self._settings + + def __enter__(self) -> "PostgresConnection": + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + +@lru_cache(maxsize=1) +def get_settings() -> DbSettings: + """Get database settings from application config.""" + from dotenv import load_dotenv + from loguru import logger + + load_dotenv() + + try: + # Temporarily disable verbose config loading logs + logger.disable("src.app.runtime") + os.environ["APP_ENVIRONMENT"] = "production" + + # Use centralized path to config.yaml + config_path = DEFAULT_PATHS.config_yaml + + if not config_path.exists(): + msg = f"Could not find config.yaml at {config_path}. Please ensure config.yaml exists in project root." + raise FileNotFoundError(msg) + + # Load config from the found path + config = load_config(file_path=config_path) + config.database.environment_mode = "production" + db_config = config.database + settings = DbSettings.load(db_config) + finally: + logger.enable("src.app.runtime") + + return settings + + +def with_postgres_connection( + settings: DbSettings | None = None, + *, + superuser_mode: bool = False, + ssl_mode: Literal["disable", "require"] = "require", +) -> Callable[[Callable[..., T]], Callable[..., T]]: + """Decorator that provides a PostgresConnection to the decorated function. + + The connection is automatically created and closed. The decorated function + must accept a 'conn' or 'connection' parameter. + + Args: + settings: Database settings (if None, will call get_settings()) + superuser_mode: If True, connect as superuser + ssl_mode: SSL mode for connection (default: require) + + Returns: + Decorator function + + Example: + >>> @with_postgres_connection() + ... def list_users(conn): + ... return conn.execute("SELECT * FROM users") + ... + >>> users = list_users() # Connection auto-created and closed + + >>> @with_postgres_connection(superuser_mode=True) + ... def create_database(conn, db_name: str): + ... conn.execute_script(f"CREATE DATABASE {db_name}") + ... + >>> create_database("newdb") + """ + + def decorator(func: Callable[..., T]) -> Callable[..., T]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> T: + # Get settings if not provided + db_settings = settings if settings is not None else get_settings() + + # Create connection within context manager + with PostgresConnection( + settings=db_settings, + superuser_mode=superuser_mode, + ssl_mode=ssl_mode, + ) as conn: + # Try to pass as keyword argument first + if "conn" not in kwargs and "connection" not in kwargs: + # Check if function signature has 'conn' or 'connection' parameter + import inspect + + sig = inspect.signature(func) + if "conn" in sig.parameters: + kwargs["conn"] = conn + elif "connection" in sig.parameters: + kwargs["connection"] = conn + else: + # Fall back to positional argument + args = (conn, *args) + + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/src/infra/postgres/init.py b/src/infra/postgres/init.py new file mode 100644 index 0000000..68157d5 --- /dev/null +++ b/src/infra/postgres/init.py @@ -0,0 +1,278 @@ +"""PostgreSQL database initialization. + +Provides database initialization including creating roles, databases, +schemas, and setting up privileges. Used by the CLI `db init` command. +""" + +from src.cli.deployment.status_display import is_temporal_enabled +from src.cli.shared.console import console +from src.infra.k8s.helpers import get_namespace, get_postgres_label + +from .connection import PostgresConnection + +K8S_NAMESPACE = get_namespace() +POSTGRES_LABEL = get_postgres_label() + + +class PostgresInitializer: + """Initializes PostgreSQL database with roles, schemas, and privileges.""" + + def __init__(self, connection: PostgresConnection) -> None: + self._connection = connection + self._settings = connection.settings + self._console = console + + def initialize(self) -> bool: + """Initialize the database with roles and schema. + + Returns: + True if initialization succeeded + """ + s = self._settings + s.ensure_all_passwords() + self._console.print("\n[bold]== Initializing PostgreSQL Database ==[/bold]") + + # Connect as superuser + conn = self._connection + + success, msg = conn.test_connection(database=self._settings.postgres_db) + if not success: + self._console.error(f"Cannot connect to PostgreSQL: {msg}") + return False + + # Show connected user + current_user = conn.scalar("SELECT current_user") + self._console.ok(f"Connected to PostgreSQL as {current_user}") + + try: + # Create roles + self._create_role(conn, s.owner_user, login=False) + self._create_role(conn, s.user, login=True, password=s.password) + self._create_role(conn, s.ro_user, login=True, password=s.ro_user_password) + + # Create database + self._create_database(conn, s.app_db, s.owner_user) + + # Set up schema and privileges + self._setup_schema_and_privileges(conn) + + # Initialize Temporal if enabled + if is_temporal_enabled() and s.temporal_password: + self._initialize_temporal(conn, s.temporal_password) + + self._console.ok("Database initialization complete!") + return True + + except Exception as e: + self._console.error(f"Initialization failed: {e}") + return False + + def _create_role( + self, + conn: PostgresConnection, + role_name: str, + login: bool, + password: str | None = None, + ) -> None: + """Create a role if it doesn't exist.""" + exists = conn.scalar( + "SELECT COUNT(*) FROM pg_roles WHERE rolname = %s", + (role_name,), + database=self._settings.postgres_db, + ) + if exists: + self._console.info(f"Role {role_name} already exists") + if password: + conn.execute_script( + f"ALTER ROLE {role_name} WITH PASSWORD '{password}'", + database=self._settings.postgres_db, + ) + self._console.ok(f"Updated password for {role_name}") + return + + login_str = "LOGIN" if login else "NOLOGIN" + password_str = f"PASSWORD '{password}'" if password else "" + conn.execute_script( + f"CREATE ROLE {role_name} WITH {login_str} {password_str}", + database=self._settings.postgres_db, + ) + self._console.ok(f"Created role {role_name}") + + def _create_database( + self, conn: PostgresConnection, db_name: str, owner: str + ) -> None: + """Create database if it doesn't exist.""" + exists = conn.scalar( + "SELECT COUNT(*) FROM pg_database WHERE datname = %s", + (db_name,), + database=self._settings.postgres_db, + ) + if exists: + self._console.info(f"Database {db_name} already exists") + return + + conn.execute_script( + f"CREATE DATABASE {db_name} OWNER {owner}", + database=self._settings.postgres_db, + ) + self._console.ok(f"Created database {db_name}") + + def _setup_schema_and_privileges(self, conn: PostgresConnection) -> None: + """Create schema and set up privileges.""" + s = self._settings + schema = "app" + + # Enable btree_gin extension (required for advanced indexing) + conn.execute_script( + "CREATE EXTENSION IF NOT EXISTS btree_gin;", + database=s.app_db, + ) + self._console.ok("Enabled btree_gin extension") + + # Create schema (connect to app database) + schema_exists = conn.scalar( + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = %s", + (schema,), + database=s.app_db, + ) + if not schema_exists: + conn.execute_script( + f"CREATE SCHEMA {schema} AUTHORIZATION {s.owner_user}", + database=s.app_db, + ) + self._console.ok(f"Created schema {schema}") + else: + self._console.info(f"Schema {schema} already exists") + + # Lock down database and schema (match shell script behavior) + conn.execute_script( + f""" + REVOKE CREATE ON DATABASE {s.app_db} FROM PUBLIC; + REVOKE ALL ON SCHEMA {schema} FROM PUBLIC; + """, + database=s.app_db, + ) + + # Grant privileges (as superuser) - USAGE + CREATE for app user + conn.execute_script( + f""" + GRANT USAGE, CREATE ON SCHEMA {schema} TO {s.user}; + GRANT USAGE ON SCHEMA {schema} TO {s.ro_user}; + + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {schema} TO {s.user}; + GRANT SELECT ON ALL TABLES IN SCHEMA {schema} TO {s.ro_user}; + + GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA {schema} TO {s.user}; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA {schema} TO {s.ro_user}; + + GRANT CONNECT ON DATABASE {s.app_db} TO {s.user}; + GRANT CONNECT ON DATABASE {s.app_db} TO {s.ro_user}; + """, + database=s.app_db, + ) + + # Set default privileges for future objects created by owner role + conn.execute_script( + f""" + ALTER DEFAULT PRIVILEGES FOR ROLE {s.owner_user} IN SCHEMA {schema} + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {s.user}; + ALTER DEFAULT PRIVILEGES FOR ROLE {s.owner_user} IN SCHEMA {schema} + GRANT SELECT ON TABLES TO {s.ro_user}; + ALTER DEFAULT PRIVILEGES FOR ROLE {s.owner_user} IN SCHEMA {schema} + GRANT USAGE, SELECT ON SEQUENCES TO {s.user}; + ALTER DEFAULT PRIVILEGES FOR ROLE {s.owner_user} IN SCHEMA {schema} + GRANT SELECT ON SEQUENCES TO {s.ro_user}; + """, + database=s.app_db, + ) + + # Set default privileges for future objects created by app user itself + # (When appuser creates tables directly via SQLModel) + conn.execute_script( + f""" + ALTER DEFAULT PRIVILEGES FOR ROLE {s.user} IN SCHEMA {schema} + GRANT SELECT ON TABLES TO {s.ro_user}; + ALTER DEFAULT PRIVILEGES FOR ROLE {s.user} IN SCHEMA {schema} + GRANT SELECT ON SEQUENCES TO {s.ro_user}; + """, + database=s.app_db, + ) + + self._console.ok(f"Set up privileges for {s.user} and {s.ro_user}") + + def _initialize_temporal( + self, conn: PostgresConnection, temporal_password: str + ) -> None: + """Initialize Temporal database roles and databases. + + Note: Temporal uses the 'public' schema (not custom schemas). + This matches the behavior of 01-init-app.sh. + """ + s = self._settings + self._console.info("Initializing Temporal database...") + + # Create roles + self._create_role(conn, s.temporal_owner, login=False) + self._create_role(conn, s.temporal_user, login=True, password=temporal_password) + + # Create databases + for db in [s.temporal_db, s.temporal_vis_db]: + self._create_database(conn, db, s.temporal_owner) + + # Configure temporal database + self._setup_temporal_database(conn, s.temporal_db) + + # Configure temporal_visibility database + self._setup_temporal_database(conn, s.temporal_vis_db) + + self._console.ok("Set up Temporal database privileges") + + def _setup_temporal_database(self, conn: PostgresConnection, db_name: str) -> None: + """Set up Temporal database with proper permissions on public schema.""" + s = self._settings + + # Enable btree_gin extension (required for Temporal advanced indexing) + conn.execute_script( + "CREATE EXTENSION IF NOT EXISTS btree_gin;", + database=db_name, + ) + self._console.ok(f"Enabled btree_gin extension in {db_name}") + + # Lock down database + conn.execute_script( + f""" + REVOKE CREATE ON DATABASE {db_name} FROM PUBLIC; + """, + database=db_name, + ) + + # Grant temporal user ability to create and use objects in public schema + conn.execute_script( + f""" + GRANT USAGE, CREATE ON SCHEMA public TO {s.temporal_user}; + """, + database=db_name, + ) + + # Default privileges for future objects owned by temporal owner + conn.execute_script( + f""" + ALTER DEFAULT PRIVILEGES FOR ROLE {s.temporal_owner} IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER ON TABLES TO {s.temporal_user}; + ALTER DEFAULT PRIVILEGES FOR ROLE {s.temporal_owner} IN SCHEMA public + GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO {s.temporal_user}; + """, + database=db_name, + ) + + # Grant privileges on existing objects (if any) + conn.execute_script( + f""" + GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER + ON ALL TABLES IN SCHEMA public TO {s.temporal_user}; + GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO {s.temporal_user}; + """, + database=db_name, + ) + + self._console.ok(f"Configured Temporal database: {db_name}") diff --git a/src/infra/postgres/migrations.py b/src/infra/postgres/migrations.py new file mode 100644 index 0000000..1635180 --- /dev/null +++ b/src/infra/postgres/migrations.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import os +import subprocess + +from src.utils.console_like import ConsoleLike +from src.utils.paths import get_project_root + + +def run_migration( + *, + action: str, + revision: str | None, + message: str | None, + merge_revisions: list[str], + purge: bool, + autogenerate: bool, + sql: bool, + database_url: str, + console: ConsoleLike, +) -> bool: + """Run Alembic migration command. + + This function is called inside a port-forward context (if bundled postgres). + """ + project_root = get_project_root() + alembic_ini = project_root / "alembic.ini" + + if not alembic_ini.exists(): + console.error(f"Alembic configuration not found: {alembic_ini}") + console.print("\n[dim]Initialize Alembic with:[/dim]") + console.print(" alembic init migrations") + return False + + env = os.environ.copy() + env["DATABASE_URL"] = database_url + + # Build alembic command + alembic_args = ["alembic", "-c", str(alembic_ini)] + + if sql and action not in {"upgrade", "downgrade"}: + console.error("--sql is only supported for upgrade/downgrade") + return False + + if action == "upgrade": + target = revision or "head" + alembic_args.extend(["upgrade", target]) + if sql: + alembic_args.append("--sql") + console.info(f"Applying migrations to: {target}") + + elif action == "downgrade": + if not revision: + console.error("Downgrade requires a target revision") + console.print("\n[dim]Examples:[/dim]") + console.print(" ... db migrate downgrade abc123") + console.print(" ... db migrate downgrade -1") + return False + alembic_args.extend(["downgrade", revision]) + if sql: + alembic_args.append("--sql") + console.warn(f"Rolling back to: {revision}") + + elif action == "current": + alembic_args.append("current") + console.info("Showing current migration state...") + + elif action == "history": + alembic_args.extend(["history", "--verbose"]) + console.info("Showing migration history...") + + elif action == "heads": + alembic_args.extend(["heads", "--verbose"]) + console.info("Showing current migration heads...") + + elif action == "show": + if not revision: + console.error("Show requires a revision") + console.print("\n[dim]Example:[/dim]") + console.print(" ... db migrate show 19becf30b774") + return False + alembic_args.extend(["show", revision]) + console.info(f"Showing migration: {revision}") + + elif action == "stamp": + if not revision: + console.error("Stamp requires a target revision") + console.print("\n[dim]Examples:[/dim]") + console.print(" ... db migrate stamp head") + console.print(" ... db migrate stamp 19becf30b774") + return False + alembic_args.extend(["stamp", revision]) + if purge: + alembic_args.append("--purge") + console.warn( + "Stamping the database (no migrations executed). " + "Use only when you understand the implications." + ) + + elif action == "merge": + merge_message = message or revision or "merge heads" + revisions_to_merge = merge_revisions or ["heads"] + alembic_args.extend(["merge", "-m", merge_message, *revisions_to_merge]) + console.info( + f"Creating merge migration: {merge_message} " + f"(revisions: {', '.join(revisions_to_merge)})" + ) + + elif action == "revision": + if not revision: + console.error("Revision requires a message") + console.print("\n[dim]Example:[/dim]") + console.print(' ... db migrate revision "add user table"') + return False + alembic_args.extend(["revision", "-m", revision]) + if autogenerate: + alembic_args.append("--autogenerate") + console.info(f"Generating migration: {revision} (with autogeneration)") + else: + console.info(f"Creating empty migration: {revision}") + + else: + console.error(f"Unknown action: {action}") + console.print( + "\n[dim]Valid actions: upgrade, downgrade, current, history, revision, " + "heads, merge, show, stamp[/dim]" + ) + return False + + # Run alembic command + result = subprocess.run( + alembic_args, + capture_output=True, + text=True, + cwd=str(project_root), + env=env, + ) + + if result.returncode != 0: + console.error(f"Migration failed:\n{result.stderr}") + if result.stdout: + console.print(result.stdout) + return False + + # Show output (stdout and stderr, as Alembic uses both) + if result.stdout.strip(): + console.print(result.stdout) + if result.stderr.strip(): + console.print(result.stderr) + + # If no output, provide helpful message based on action + if not result.stdout.strip() and not result.stderr.strip(): + if action == "current": + console.print("[dim]No migrations applied yet.[/dim]") + elif action == "history": + console.print("[dim]No migration history found.[/dim]") + + if action == "upgrade": + console.ok("Migrations applied successfully") + elif action == "downgrade": + console.ok("Rollback completed successfully") + elif action == "revision": + console.ok("Migration file created successfully") + console.print("\n[dim]Next steps:[/dim]") + console.print(" 1. Review the generated migration in migrations/versions/") + console.print(" 2. Run '... db migrate upgrade' to apply it") + elif action == "merge": + console.ok("Merge migration created successfully") + + return True diff --git a/src/infra/postgres/reset.py b/src/infra/postgres/reset.py new file mode 100644 index 0000000..970d1ea --- /dev/null +++ b/src/infra/postgres/reset.py @@ -0,0 +1,179 @@ +"""PostgreSQL database reset functionality. + +Provides database-level reset operations to return PostgreSQL to a clean state. +This drops all databases, roles, and schemas created by the application. +""" + +from src.cli.shared.console import console +from src.infra.k8s.helpers import get_namespace, get_postgres_label +from src.infra.k8s.port_forward import with_postgres_port_forward_if_needed + +from .connection import PostgresConnection + +K8S_NAMESPACE = get_namespace() +POSTGRES_LABEL = get_postgres_label() + + +class PostgresReset: + """Resets PostgreSQL database to clean state. + + This includes: + - Dropping application databases + - Dropping application roles + - Dropping application schemas + - Preserving system databases and roles + """ + + def __init__(self, connection: PostgresConnection) -> None: + self._connection = connection + self._settings = connection.settings + self._console = console + + @with_postgres_port_forward_if_needed( + namespace=K8S_NAMESPACE, pod_label=POSTGRES_LABEL + ) + def reset(self, include_temporal: bool = True) -> bool: + """Reset the PostgreSQL database to clean state. + + Drops all application-created databases, roles, and schemas. + + Args: + superuser_password: Password for superuser connection + include_temporal: Whether to drop Temporal databases/roles + + Returns: + True if reset succeeded, False otherwise + """ + self._console.print("\n[bold]== PostgreSQL Database Reset ==[/bold]") + + try: + # Connect as superuser to postgres database + self._settings.ensure_superuser_password() + conn = self._connection + + # Test connection + success, msg = conn.test_connection(database="postgres") + if not success: + self._console.error(f"Cannot connect to PostgreSQL: {msg}") + return False + + self._console.ok("Connected to PostgreSQL as superuser") + + # Terminate connections to databases we're about to drop + self._terminate_connections(conn, include_temporal) + + # Drop application database + self._drop_database(conn, self._settings.app_db) + + # Drop Temporal databases if requested + if include_temporal: + self._drop_database(conn, self._settings.temporal_db) + self._drop_database(conn, self._settings.temporal_vis_db) + + # Drop application roles + self._drop_role(conn, self._settings.user) + self._drop_role(conn, self._settings.ro_user) + self._drop_role(conn, self._settings.owner_user) + + # Drop Temporal roles if requested + if include_temporal: + self._drop_role(conn, self._settings.temporal_user) + self._drop_role(conn, self._settings.temporal_owner) + + self._console.ok("PostgreSQL database reset complete!") + return True + + except Exception as e: + self._console.error(f"Reset failed: {e}") + return False + + def _terminate_connections( + self, conn: PostgresConnection, include_temporal: bool + ) -> None: + """Terminate all connections to application databases.""" + s = self._settings + databases = [s.app_db] + + if include_temporal: + databases.extend([s.temporal_db, s.temporal_vis_db]) + + for db in databases: + try: + conn.execute_script( + f""" + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '{db}' + AND pid <> pg_backend_pid(); + """, + database="postgres", + ) + self._console.info(f"Terminated connections to {db}") + except Exception: + # Database might not exist, ignore + pass + + def _drop_database(self, conn: PostgresConnection, db_name: str) -> None: + """Drop a database if it exists.""" + try: + exists = conn.scalar( + "SELECT COUNT(*) FROM pg_database WHERE datname = %s", + (db_name,), + database="postgres", + ) + + if exists: + # Terminate connections using a DO block to avoid SELECT result issues + conn.execute_script( + f""" + DO $$ + DECLARE + r RECORD; + BEGIN + FOR r IN + SELECT pid FROM pg_stat_activity + WHERE datname = '{db_name}' AND pid <> pg_backend_pid() + LOOP + PERFORM pg_terminate_backend(r.pid); + END LOOP; + END $$; + """, + database="postgres", + ) + + # Drop database (already in autocommit mode via execute_script) + conn.execute_script( + f"DROP DATABASE IF EXISTS {db_name}", database="postgres" + ) + self._console.ok(f"Dropped database: {db_name}") + else: + self._console.info(f"Database {db_name} does not exist") + + except Exception as e: + self._console.warn(f"Could not drop database {db_name}: {e}") + + def _drop_role(self, conn: PostgresConnection, role_name: str) -> None: + """Drop a role if it exists.""" + try: + exists = conn.scalar( + "SELECT COUNT(*) FROM pg_roles WHERE rolname = %s", + (role_name,), + database="postgres", + ) + + if exists: + # Reassign owned objects and drop role + conn.execute_script( + f""" + REASSIGN OWNED BY {role_name} TO {self._settings.superuser}; + DROP OWNED BY {role_name}; + DROP ROLE IF EXISTS {role_name}; + """, + database="postgres", + ) + self._console.ok(f"Dropped role: {role_name}") + else: + self._console.info(f"Role {role_name} does not exist") + + except Exception as e: + self._console.warn(f"Could not drop role {role_name}: {e}") diff --git a/src/infra/postgres/sync.py b/src/infra/postgres/sync.py new file mode 100644 index 0000000..faf41de --- /dev/null +++ b/src/infra/postgres/sync.py @@ -0,0 +1,179 @@ +"""PostgreSQL password synchronization. + +Syncs database user role names and passwords from secret files to PostgreSQL. +""" + +from pathlib import Path + +import psycopg2 + +from src.cli.deployment.base import BaseDeployer +from src.cli.deployment.status_display import is_temporal_enabled +from src.cli.shared.console import console +from src.infra.constants import DEFAULT_CONSTANTS +from src.infra.utils.service_config import is_bundled_postgres_enabled + +from .connection import PostgresConnection + + +class PostgresPasswordSync: + """Synchronizes PostgreSQL user passwords from secret files. + + For both Docker Compose and Kubernetes deployments. + """ + + def __init__( + self, + connection: PostgresConnection, + deployer: BaseDeployer, + secrets_dirs: list[Path], + ) -> None: + self._deployer = deployer + self._settings = connection.settings + self._connection = connection + self._console = console + self._secrets_dirs = secrets_dirs + + def _read_secret(self, filename: str) -> str | None: + """Read a secret from file. + + Tries secrets_dir first, then keys_dir. + """ + for base in self._secrets_dirs: + path = base / filename + if path.exists(): + return path.read_text().strip() + return None + + def sync_bundled_superuser_password(self) -> bool: + """Sync the bundled PostgreSQL superuser password by deploying secrets and restarting the bundled postgres pod/container. + + The container/pod will pick up the new password on restart. + + Returns: + True if sync succeeded + """ + if not is_bundled_postgres_enabled(): + self._console.info( + "Bundled PostgreSQL is not enabled; skipping superuser password sync" + ) + return True + + # Step 1: Deploy secrets to K8s so the pod can pick them up on restart + if not self._deployer.deploy_secrets(): + return False + + # Step 2: Restart PostgreSQL to pick up new secrets + # This is done BEFORE port-forwarding since it kills the pod + if not self._deployer.restart_resource( + label=DEFAULT_CONSTANTS.POSTGRES_RESOURCE_NAME, + resource_type="statefulset", + timeout=120, + ): + return False + + return True + + def sync_user_roles_and_passwords(self) -> bool: + """Run initialization after PostgreSQL restart with fresh port-forward.""" + from src.infra.postgres import PostgresInitializer + + self._console.info( + "Re-running idempotent initialization to sync roles and passwords" + ) + # Step 1: Re-run initialization + initializer = PostgresInitializer(connection=self._connection) + success = initializer.initialize() + + if success: + self._console.info("Fixing database and schema ownership if needed") + if not self._fix_ownership(): + success = False + + return success + + def _fix_ownership(self) -> bool: + """Fix ownership of databases and schemas to use correct owner roles. + + Returns: + True if ownership was fixed successfully + """ + s = self._settings + conn = self._connection + success = True + + try: + # Fix app database ownership + self._console.info(f"Ensuring {s.app_db} is owned by {s.owner_user}") + conn.execute_script( + f"ALTER DATABASE {s.app_db} OWNER TO {s.owner_user}", + database=self._settings.postgres_db, + ) + + # Fix app schema ownership + self._console.info(f"Ensuring schema app is owned by {s.owner_user}") + conn.execute_script( + f"ALTER SCHEMA app OWNER TO {s.owner_user}", + database=s.app_db, + ) + + if is_temporal_enabled(): + # Fix Temporal database ownership + self._console.info( + f"Ensuring {s.temporal_db} is owned by {s.temporal_owner}" + ) + conn.execute_script( + f"ALTER DATABASE {s.temporal_db} OWNER TO {s.temporal_owner}", + database=self._settings.postgres_db, + ) + + self._console.info( + f"Ensuring {s.temporal_vis_db} is owned by {s.temporal_owner}" + ) + conn.execute_script( + f"ALTER DATABASE {s.temporal_vis_db} OWNER TO {s.temporal_owner}", + database=self._settings.postgres_db, + ) + + self._console.ok("Fixed database and schema ownership") + return success + + except Exception as e: + self._console.error(f"Failed to fix ownership: {e}") + return False + + def verify_password( + self, + user: str, + password: str, + database: str | None = None, + ) -> bool: + """Verify a user can connect with the given password. + + Args: + user: Username to test + password: Password to test + database: Database to connect to + + Returns: + True if connection succeeds + """ + s = self._settings + db = database or s.app_db + + try: + with psycopg2.connect( + host=s.host, + port=s.port, + dbname=db, + user=user, + password=password, + connect_timeout=5, + ) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + self._console.ok(f"Verified {user} can connect to {db}") + return True + except psycopg2.OperationalError as e: + self._console.error(f"Verification failed for {user}: {e}") + return False diff --git a/src/infra/postgres/verify.py b/src/infra/postgres/verify.py new file mode 100644 index 0000000..daa2909 --- /dev/null +++ b/src/infra/postgres/verify.py @@ -0,0 +1,416 @@ +"""PostgreSQL database verification. + +Provides verification of PostgreSQL setup including roles, permissions, +and TLS configuration. +""" + +from dataclasses import dataclass, field +from enum import Enum + +from rich.table import Table + +from src.cli.shared.console import console +from src.infra.k8s.helpers import get_namespace, get_postgres_label +from src.infra.utils.service_config import is_temporal_enabled + +from .connection import PostgresConnection + +K8S_NAMESPACE = get_namespace() +POSTGRES_LABEL = get_postgres_label() + + +class CheckStatus(Enum): + """Status of a verification check.""" + + PASS = "pass" + FAIL = "fail" + WARN = "warn" + SKIP = "skip" + + +@dataclass +class CheckResult: + """Result of a single verification check.""" + + name: str + status: CheckStatus + message: str + details: str | None = None + + +class PostgresVerifier: + """Verifies PostgreSQL database setup and configuration.""" + + def __init__(self, connection: PostgresConnection) -> None: + self._settings = connection.settings + self._connection = connection + self._console = console + self._results: list[CheckResult] = field(default_factory=list) + + # Password for connecting (prompted by CLI) + password: str = "" + user_name: str = "" + + def _ok(self, name: str, message: str) -> None: + self._results.append(CheckResult(name, CheckStatus.PASS, message)) + self._console.ok(f" {message}") + + def _bad(self, name: str, message: str, details: str | None = None) -> None: + self._results.append(CheckResult(name, CheckStatus.FAIL, message, details)) + self._console.error(f" {message}") + + def _warn(self, name: str, message: str, details: str | None = None) -> None: + self._results.append(CheckResult(name, CheckStatus.WARN, message, details)) + self._console.warn(f" {message}") + + def _skip(self, name: str, message: str) -> None: + self._results.append(CheckResult(name, CheckStatus.SKIP, message)) + self._console.print(f"[dim]⏭️ {message}[/dim]") + + def verify_all(self) -> bool: + """Run all verification checks. + + Returns: + True if all critical checks pass + """ + self._results = [] + s = self._settings + s.ensure_superuser_password() + + conn = self._connection + + self._console.print("\n[bold]== Verifying PostgreSQL Configuration ==[/bold]") + self._console.print(f" Host: {s.host}:{s.port}") + self._console.print(f" DB={s.app_db} OWNER={s.owner_user}") + self._console.print(f" USER={s.user} RO={s.ro_user}") + self._console.print() + + # Test connection + success, msg = conn.test_connection() + if not success: + self._bad("connection", f"Cannot connect: {msg}") + return False + self._ok("connection", "Connected to PostgreSQL") + + # Run checks + self._verify_roles(conn) + self._verify_passwords(conn) + self._verify_database_ownership(conn) + self._verify_schema(conn) + self._verify_schema_permissions(conn) + self._verify_table_privileges(conn) + self._verify_tls(conn) + + # Summary + failed = [r for r in self._results if r.status == CheckStatus.FAIL] + warnings = [r for r in self._results if r.status == CheckStatus.WARN] + + self._console.print() + if failed: + self._console.print(f"[red]❌ {len(failed)} check(s) failed[/red]") + return False + if warnings: + self._console.print( + f"[yellow]⚠️ Passed with {len(warnings)} warning(s)[/yellow]" + ) + else: + self._console.print("[green]✅ All checks passed 🎉[/green]") + return True + + def _verify_roles(self, conn: PostgresConnection) -> None: + """Verify database roles exist with correct attributes.""" + s = self._settings + roles = [ + (s.user, True, "app user"), + (s.ro_user, True, "read-only user"), + (s.owner_user, False, "owner role"), + ] + + if is_temporal_enabled(): + roles.extend( + [ + (s.temporal_user, True, "Temporal app user"), + (s.temporal_owner, False, "Temporal owner role"), + ] + ) + + for role_name, should_login, desc in roles: + count = conn.scalar( + "SELECT COUNT(*) FROM pg_roles WHERE rolname = %s", (role_name,) + ) + if not count: + self._bad(f"role_{role_name}", f"Role {role_name} ({desc}) missing") + continue + + can_login = conn.scalar( + "SELECT rolcanlogin FROM pg_roles WHERE rolname = %s", (role_name,) + ) + login_str = "LOGIN" if should_login else "NOLOGIN" + + if should_login and not can_login: + self._bad(f"role_{role_name}", f"{role_name} should have LOGIN") + elif not should_login and can_login: + self._warn(f"role_{role_name}", f"{role_name} has LOGIN but shouldn't") + else: + self._ok(f"role_{role_name}", f"Role {role_name} ({login_str})") + + def _verify_passwords(self, conn: PostgresConnection) -> None: + """Verify that passwords in local files match what's in the database. + + Attempts to connect as each user role with the password from the local file. + This ensures the local secrets are in sync with the database. + """ + s = self._settings + + # Only verify password for roles that should be able to login + roles_to_test = [ + (s.user, s.password, "app user"), + (s.ro_user, s.ro_user_password, "read-only user"), + ] + + if is_temporal_enabled(): + roles_to_test.append( + (s.temporal_user, s.temporal_password, "Temporal user") + ) + + for role_name, password, desc in roles_to_test: + if not password: + self._warn( + f"password_{role_name}", + f"No password available for {role_name} ({desc}) - skipping password verification", + ) + continue + + # Try to connect as this user with the password from the local file + try: + import psycopg2 + + test_conn = conn.get_dsn() + test_conn["user"] = role_name + test_conn["password"] = password + + test_conn = psycopg2.connect(**test_conn) + test_conn.close() + + self._ok( + f"password_{role_name}", + f"Password verified for {role_name} ({desc})", + ) + except psycopg2.OperationalError as e: + if "password authentication failed" in str(e): + self._bad( + f"password_{role_name}", + f"Password mismatch for {role_name} ({desc})", + details=( + "The password in your local secrets file does not match what's in the database.\n" + "Fix: Run 'uv run api-forge-cli k8s db sync' to update database passwords." + ), + ) + else: + self._warn( + f"password_{role_name}", + f"Could not verify password for {role_name}: {e}", + ) + except Exception as e: + self._warn( + f"password_{role_name}", + f"Unexpected error verifying password for {role_name}: {e}", + ) + + def _verify_database_ownership(self, conn: PostgresConnection) -> None: + """Verify database ownership.""" + s = self._settings + owner = conn.scalar( + """ + SELECT pg_catalog.pg_get_userbyid(d.datdba) + FROM pg_catalog.pg_database d WHERE d.datname = %s + """, + (s.app_db,), + ) + if not owner: + self._bad("db_ownership", f"Database {s.app_db} does not exist") + elif owner != s.owner_user: + self._bad( + "db_ownership", f"{s.app_db} owned by {owner}, expected {s.owner_user}" + ) + else: + self._ok("db_ownership", f"Database {s.app_db} owned by {s.owner_user}") + + def _verify_schema(self, conn: PostgresConnection) -> None: + """Verify schema ownership.""" + s = self._settings + schema = "app" + owner = conn.scalar( + "SELECT nspowner::regrole::text FROM pg_namespace WHERE nspname = %s", + (schema,), + s.app_db, + ) + if not owner: + self._bad("schema", f"Schema {schema} does not exist") + elif owner != s.owner_user: + self._warn( + "schema", f"Schema {schema} owned by {owner}, expected {s.owner_user}" + ) + else: + self._ok("schema", f"Schema {schema} owned by {s.owner_user}") + + def _verify_schema_permissions(self, conn: PostgresConnection) -> None: + """Verify schema-level permissions for users.""" + s = self._settings + + # Check app user permissions on app schema + self._check_schema_user_permissions( + conn, s.user, "app", s.app_db, usage=True, create=True, desc="app user" + ) + + # Check if Temporal is enabled + # Note: Temporal uses the 'public' schema, not custom schemas + if is_temporal_enabled(): + # Check temporal user permissions on public schema in temporal databases + # Each database has its own public schema with separate permissions + self._check_schema_user_permissions( + conn, + s.temporal_user, + "public", + s.temporal_db, + usage=True, + create=True, + desc=f"Temporal user on {s.temporal_db}.public", + ) + self._check_schema_user_permissions( + conn, + s.temporal_user, + "public", + s.temporal_vis_db, + usage=True, + create=True, + desc=f"Temporal user on {s.temporal_vis_db}.public", + ) + + def _check_schema_user_permissions( + self, + conn: PostgresConnection, + user: str, + schema: str, + database: str, + usage: bool, + create: bool, + desc: str, + ) -> None: + """Check specific user permissions on a schema. + + Args: + conn: Database connection + user: Username to check permissions for + schema: Schema name + database: Database name containing the schema + usage: Whether USAGE permission is expected + create: Whether CREATE permission is expected + desc: Description for logging + """ + # Check if schema exists + schema_exists = conn.scalar( + "SELECT COUNT(*) FROM pg_namespace WHERE nspname = %s", + (schema,), + database, + ) + + if not schema_exists: + self._bad( + f"schema_perms_{user}_{schema}", + f"Schema {schema} does not exist in {database}", + ) + return + + # Check USAGE privilege + has_usage = conn.scalar( + "SELECT has_schema_privilege(%s, %s, 'USAGE')", + (user, schema), + database, + ) + + # Check CREATE privilege + has_create = conn.scalar( + "SELECT has_schema_privilege(%s, %s, 'CREATE')", + (user, schema), + database, + ) + + issues = [] + if usage and not has_usage: + issues.append("USAGE") + if create and not has_create: + issues.append("CREATE") + + if issues: + self._bad( + f"schema_perms_{user}_{schema}", + f"{desc} missing {', '.join(issues)} on schema {schema} in {database}", + ) + else: + perms = [] + if has_usage: + perms.append("USAGE") + if has_create: + perms.append("CREATE") + self._ok( + f"schema_perms_{user}_{schema}", + f"{desc} has {', '.join(perms)} on schema {schema}", + ) + + def _verify_table_privileges(self, conn: PostgresConnection) -> None: + """Verify table privileges.""" + s = self._settings + schema = "app" + + table_count = conn.scalar( + "SELECT COUNT(*) FROM pg_tables WHERE schemaname = %s", + (schema,), + s.app_db, + ) + if not table_count: + self._skip( + "table_privileges", + f"No tables in schema {schema} (created when app runs)", + ) + return + + has_privs = conn.scalar( + """ + SELECT COUNT(*) FROM information_schema.table_privileges + WHERE grantee = %s AND table_schema = %s AND privilege_type = 'SELECT' + """, + (s.user, schema), + s.app_db, + ) + if has_privs: + self._ok("table_privileges", f"{s.user} has table privileges") + else: + self._warn("table_privileges", f"{s.user} may be missing privileges") + + def _verify_tls(self, conn: PostgresConnection) -> None: + """Verify TLS configuration.""" + ssl_mode = conn.scalar("SHOW ssl") + if ssl_mode == "on": + self._ok("tls", "SSL is enabled") + else: + self._warn("tls", f"SSL is {ssl_mode} (expected 'on' for production)") + + def print_summary(self) -> None: + """Print summary table of results.""" + table = Table(title="Verification Summary") + table.add_column("Check", style="cyan") + table.add_column("Status") + table.add_column("Message") + + status_map = { + CheckStatus.PASS: "[green]✅ PASS[/green]", + CheckStatus.FAIL: "[red]❌ FAIL[/red]", + CheckStatus.WARN: "[yellow]⚠️ WARN[/yellow]", + CheckStatus.SKIP: "[dim]⏭️ SKIP[/dim]", + } + + for r in self._results: + table.add_row(r.name, status_map[r.status], r.message) + + self._console.print(table) diff --git a/src/cli/deployment/service_config.py b/src/infra/utils/service_config.py similarity index 65% rename from src/cli/deployment/service_config.py rename to src/infra/utils/service_config.py index 127cc9c..dafa0cc 100644 --- a/src/cli/deployment/service_config.py +++ b/src/infra/utils/service_config.py @@ -57,15 +57,48 @@ def is_temporal_enabled() -> bool: return True +def is_bundled_postgres_enabled() -> bool: + """Check if bundled PostgreSQL is enabled in config.yaml. + + When bundled_postgres.enabled=False, PostgreSQL containers are not deployed + and the app connects to an external PostgreSQL via DATABASE_URL. + + Uses load_config(processed=False) to read raw YAML without environment + processing, avoiding side effects from loading the full application config. + + Returns: + True if bundled PostgreSQL is enabled, False otherwise + """ + try: + from src.app.runtime.config.config_loader import load_config + + # Load raw config without environment variable substitution or processing + config_data = load_config(processed=False) + + # Navigate to config.database.bundled_postgres.enabled in the YAML structure + config_section = config_data.get("config", {}) + database_config = config_section.get("database", {}) + bundled_postgres = database_config.get("bundled_postgres", {}) + return cast( + bool, bundled_postgres.get("enabled", True) + ) # Default to True if not specified + + except Exception: + # If we can't load config, assume bundled PostgreSQL is enabled + return True + + def get_production_services() -> list[tuple[str, str]]: """Get list of production services based on configuration. Returns: List of (container_name, display_name) tuples for active services """ - services: list[tuple[str, str]] = [ - ("api-forge-postgres", "PostgreSQL"), - ] + services: list[tuple[str, str]] = [] + + # Add PostgreSQL if bundled postgres is enabled + if is_bundled_postgres_enabled(): + services.append(("api-forge-postgres", "PostgreSQL")) # Add Redis if it's enabled in config if is_redis_enabled(): diff --git a/src/utils/console_like.py b/src/utils/console_like.py new file mode 100644 index 0000000..1e55c9c --- /dev/null +++ b/src/utils/console_like.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Protocol + +from rich.console import ConsoleRenderable + + +class ConsoleLike(Protocol): + def print(self, msg: ConsoleRenderable | str | None = None) -> None: ... + + def info(self, msg: str) -> None: ... + + def warn(self, msg: str) -> None: ... + + def error(self, msg: str) -> None: ... + + def ok(self, msg: str) -> None: ... + + +class StdoutConsole: + """Minimal console fallback. + + Keeps infrastructure code usable without importing the CLI console. + """ + + def print(self, msg: ConsoleRenderable | str | None = None) -> None: + print(msg) + + def info(self, msg: str) -> None: + print(msg) + + def warn(self, msg: str) -> None: + print(msg) + + def error(self, msg: str) -> None: + print(msg) + + def ok(self, msg: str) -> None: + print(msg) + + +def coalesce_console(console: ConsoleLike | None) -> ConsoleLike: + return console if console is not None else StdoutConsole() diff --git a/src/utils/paths.py b/src/utils/paths.py index ce61ed0..b2b7402 100644 --- a/src/utils/paths.py +++ b/src/utils/paths.py @@ -1,4 +1,27 @@ +from pathlib import Path + + def make_postgres_url( user: str, password: str, host: str, port: int, dbname: str ) -> str: return f"postgresql://{user}:{password}@{host}:{port}/{dbname}" + + +def get_project_root() -> Path: + """Get the project root directory. + + Walks up from the module location to find the project root, + identified by the presence of pyproject.toml. + + Returns: + Path to the project root directory + """ + current = Path(__file__).resolve() + + # Walk up the directory tree looking for pyproject.toml + for parent in [current, *current.parents]: + if (parent / "pyproject.toml").exists(): + return parent + + # Fallback to four levels up (src/cli/commands/shared.py -> project root) + return Path(__file__).parent.parent.parent.parent diff --git a/tests/conftest.py b/tests/conftest.py index 2bee866..e757437 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os +from pathlib import Path # Set test mode flag BEFORE any imports that might load config # This allows config loading to use test defaults instead of requiring all env vars @@ -12,4 +13,8 @@ os.environ.setdefault("SESSION_SIGNING_SECRET", "test-session-secret-32-bytes-long") os.environ.setdefault("CSRF_SIGNING_SECRET", "test-csrf-secret-32-bytes-long-") -from tests.fixtures import * # noqa: F401,F403 +# Ensure logs directory exists to prevent Loguru errors during tests +_logs_dir = Path(__file__).parent.parent / "logs" +_logs_dir.mkdir(exist_ok=True) + +from tests.fixtures import * # noqa: E402, F401, F403 diff --git a/tests/e2e/test_copier_to_deployment.py b/tests/e2e/test_copier_to_deployment.py index 4aa87bb..9de7b4e 100644 --- a/tests/e2e/test_copier_to_deployment.py +++ b/tests/e2e/test_copier_to_deployment.py @@ -126,6 +126,8 @@ def run_command( timeout: int = 300, check: bool = True, stream_output: bool = False, + env: dict[str, str] | None = None, + silent: bool = False, ) -> subprocess.CompletedProcess: """Run a command and return the result. @@ -135,13 +137,16 @@ def run_command( timeout: Command timeout in seconds check: Whether to raise exception on non-zero exit stream_output: If True, stream output in real-time (for long-running commands) + env: Environment variables (defaults to os.environ with VIRTUAL_ENV removed) + silent: If True, suppress automatic stdout/stderr printing (useful for JSON output) """ print(f"\n🔧 Running: {' '.join(cmd)}") print(f" Working directory: {cwd}") # Clear VIRTUAL_ENV to avoid "does not match project environment" warnings # when running uv commands in the generated project directory - env = os.environ.copy() + if env is None: + env = os.environ.copy() env.pop("VIRTUAL_ENV", None) if stream_output: @@ -203,10 +208,11 @@ def read_output(): env=env, ) - if result.stdout: - print(f"📤 stdout:\n{result.stdout}") - if result.stderr: - print(f"📤 stderr:\n{result.stderr}") + if not silent: + if result.stdout: + print(f"📤 stdout:\n{result.stdout}") + if result.stderr: + print(f"📤 stderr:\n{result.stderr}") if check and result.returncode != 0: raise RuntimeError( @@ -548,18 +554,7 @@ def test_07_docker_compose_prod_deployment(self): # Give it a moment to clean up time.sleep(5) - # Setup .env file (required for production deployment) - env_example = project_dir / ".env.example" - env_file = project_dir / ".env" - - if env_example.exists(): - print("📝 Creating .env from .env.example...") - shutil.copy(env_example, env_file) - else: - print("⚠️ .env.example not found, creating minimal .env...") - env_file.write_text("APP_ENVIRONMENT=production\n") - - # Ensure secrets are generated (Docker Compose needs them) + # Ensure secrets are generated FIRST (Docker Compose and init need them) # If test_06 already ran, secrets will exist # If running test_07 alone, this ensures secrets are available secrets_base = project_dir / "infra" / "secrets" @@ -634,12 +629,174 @@ def test_07_docker_compose_prod_deployment(self): else: print("✅ Secrets already exist (from test_06)") + # Debug: Verify all required password files exist + print("\n🔍 Verifying required password files...") + required_password_files = [ + "postgres_password.txt", + "postgres_app_user_pw.txt", + "postgres_app_ro_pw.txt", + "postgres_app_owner_pw.txt", + "postgres_temporal_pw.txt", + "redis_password.txt", + "session_signing_secret.txt", + "csrf_signing_secret.txt", + ] + for password_file in required_password_files: + file_path = keys_dir / password_file + if file_path.exists(): + size = file_path.stat().st_size + print(f" ✓ {password_file} ({size} bytes)") + else: + print(f" ✗ {password_file} MISSING!") + raise AssertionError( + f"Required password file missing: {password_file}" + ) + print("✅ All required password files exist") + + # Setup .env file AFTER secrets generation (so password paths are available) + env_example = project_dir / ".env.example" + env_file = project_dir / ".env" + + if env_example.exists(): + print("📝 Creating .env from .env.example...") + shutil.copy(env_example, env_file) + # Override environment to production for Docker Compose deployment + env_content = env_file.read_text() + print( + f"🔍 Before modification - APP_ENVIRONMENT line: {[line for line in env_content.split('\\n') if 'APP_ENVIRONMENT' in line]}" + ) + + if "APP_ENVIRONMENT=" in env_content: + env_content = "\n".join( + "APP_ENVIRONMENT=production" + if line.startswith("APP_ENVIRONMENT=") + else line + for line in env_content.split("\n") + ) + else: + env_content = f"APP_ENVIRONMENT=production\n{env_content}" + env_file.write_text(env_content) + + # Verify the change was written + verification = env_file.read_text() + print( + f"🔍 After modification - APP_ENVIRONMENT line: {[line for line in verification.split('\\n') if 'APP_ENVIRONMENT' in line]}" + ) + assert "APP_ENVIRONMENT=production" in verification, ( + "Failed to set APP_ENVIRONMENT=production in .env" + ) + print("✅ .env updated with APP_ENVIRONMENT=production") + else: + print("⚠️ .env.example not found, creating minimal .env...") + env_file.write_text("APP_ENVIRONMENT=production\n") + + # Debug: Show password-related env vars in .env + print("\n🔍 Checking .env for password file references...") + if env_file.exists(): + env_content = env_file.read_text() + password_vars = [ + line + for line in env_content.split("\n") + if "PASSWORD" in line and not line.strip().startswith("#") + ] + for var in password_vars[:5]: # Show first 5 + print(f" {var}") + if len(password_vars) > 5: + print(f" ... and {len(password_vars) - 5} more") + print("✅ .env file configured") + + # Debug: Check healthcheck configuration in docker-compose.prod.yml + print("\n🔍 Checking PostgreSQL healthcheck in docker-compose.prod.yml...") + compose_file = project_dir / "docker-compose.prod.yml" + if compose_file.exists(): + compose_content = compose_file.read_text() + # Extract healthcheck section + import re + + healthcheck_match = re.search( + r"healthcheck:.*?(?=\n [a-z_]+:|\n\n|\Z)", + compose_content, + re.DOTALL, + ) + if healthcheck_match: + print(f" {healthcheck_match.group(0)[:200]}...") + else: + print(" ⚠️ No healthcheck found in docker-compose.prod.yml") + + # Create and initialize bundled PostgreSQL database + print("\n🗄️ Creating bundled PostgreSQL database...") + # Pass APP_ENVIRONMENT=production via environment variable to override .env + create_env = os.environ.copy() + create_env["APP_ENVIRONMENT"] = "production" + + # Prevent secrets from the template repo leaking into the generated project. + # The generated project has its own secret files under infra/secrets/keys. + for key in list(create_env.keys()): + if key.startswith("POSTGRES_"): + create_env.pop(key, None) + + self.run_command( + ["uv", "run", "api-forge-cli", "prod", "db", "create", "--bundled"], + cwd=project_dir, + timeout=300, # 5 minutes for database creation and init + env=create_env, + ) + + # Show password from file + password_file = project_dir / "infra/secrets/keys/postgres_password.txt" + if password_file.exists(): + password_from_file = password_file.read_text().strip() + print( + f"Password in file: {password_from_file[:4]}...{password_from_file[-4:]} ({len(password_from_file)} chars)" + ) + + # Show env var in container + print("\nChecking container environment:") + result = self.run_command( + [ + "docker", + "exec", + "api-forge-postgres", + "printenv", + "POSTGRES_PASSWORD", + ], + cwd=project_dir, + check=False, + ) + if result.returncode == 0: + container_password = result.stdout.strip() + print( + f"POSTGRES_PASSWORD in container: {container_password[:4]}...{container_password[-4:]} ({len(container_password)} chars)" + ) + print(f"Passwords match: {password_from_file == container_password}") + + # Try manual psql connection + print("\nTesting psql connection:") + result = self.run_command( + [ + "docker", + "exec", + "api-forge-postgres", + "psql", + "-U", + "postgres", + "-h", + "127.0.0.1", + "-c", + "SELECT 1", + ], + cwd=project_dir, + check=False, + ) + print(f"Direct psql connection result: {result.returncode}") + # Start production deployment try: result = self.run_command( ["uv", "run", "api-forge-cli", "prod", "up"], cwd=project_dir, timeout=600, # 10 minutes for building images in CI + env=create_env, ) except RuntimeError as e: print(f"\n❌ Deployment failed: {e}") @@ -675,7 +832,6 @@ def test_07_docker_compose_prod_deployment(self): print("⏳ Waiting for services to become healthy...") time.sleep(30) - # TODO: This is for debugging; remove later # Check if temporal-schema-setup completed successfully print("\n🔍 Checking temporal-schema-setup status...") try: @@ -715,6 +871,7 @@ def test_07_docker_compose_prod_deployment(self): result = self.run_command( ["uv", "run", "api-forge-cli", "prod", "status"], cwd=project_dir, + env=create_env, ) print(f"Deployment status:\n{result.stdout}") @@ -856,13 +1013,11 @@ def test_08_kubernetes_deployment(self): # Clean up any previous K8s deployment first print("\n🧹 Cleaning up any previous K8s deployment...") self.run_command( - ["kubectl", "delete", "namespace", "api-forge-prod", "--wait=false"], + ["kubectl", "delete", "namespace", "api-forge-prod", "--wait=true"], cwd=project_dir, - timeout=30, + timeout=120, # Wait up to 2 minutes for namespace deletion check=False, ) - # Give it a moment to start deleting - time.sleep(5) # Setup .env file (required for K8s deployment) env_example = project_dir / ".env.example" @@ -930,6 +1085,26 @@ def test_08_kubernetes_deployment(self): else: print("✅ Secrets already exist (from test_06)") + # Create and initialize bundled PostgreSQL database + print("\n🗄️ Creating bundled PostgreSQL database...") + + # Sanitize environment: prevent template repo secrets from leaking + # into the generated project subprocess. The generated project has + # its own secret files under infra/secrets/keys. + create_env = os.environ.copy() + create_env["APP_ENVIRONMENT"] = "production" + for key in list(create_env.keys()): + if key.startswith("POSTGRES_"): + create_env.pop(key, None) + + self.run_command( + ["uv", "run", "api-forge-cli", "k8s", "db", "create", "--bundled"], + cwd=project_dir, + timeout=300, # 5 minutes for database creation and init + env=create_env, + ) + print("✅ PostgreSQL database created and initialized") + # Deploy to Kubernetes (with real-time output streaming) print("🚀 Starting K8s deployment with real-time output...") result = self.run_command( @@ -937,6 +1112,7 @@ def test_08_kubernetes_deployment(self): cwd=project_dir, timeout=600, stream_output=True, + env=create_env, ) # Wait for pods to be ready with retries @@ -961,6 +1137,7 @@ def test_08_kubernetes_deployment(self): ], cwd=project_dir, check=False, + silent=True, ) try: @@ -1107,26 +1284,6 @@ def test_08_kubernetes_deployment(self): print("✅ All pods deployed") - # Check postgres-verifier job completed successfully - result = self.run_command( - [ - "kubectl", - "get", - "job", - "postgres-verifier", - "-n", - "api-forge-prod", - "-o", - "jsonpath={.status.succeeded}", - ], - cwd=project_dir, - ) - - assert result.stdout == "1", ( - "postgres-verifier job did not complete successfully" - ) - print("✅ postgres-verifier job completed") - # Check worker is using correct module name result = self.run_command( [ diff --git a/tests/unit/app/core/config/test_database_config.py b/tests/unit/app/core/config/test_database_config.py index eec3b05..0e7c3fb 100644 --- a/tests/unit/app/core/config/test_database_config.py +++ b/tests/unit/app/core/config/test_database_config.py @@ -7,7 +7,7 @@ import pytest from pydantic import ValidationError -from src.app.runtime.config.config_data import DatabaseConfig +from src.app.runtime.config.config_data import BundledPostgresConfig, DatabaseConfig class TestDatabaseConfigPasswordResolution: @@ -70,7 +70,9 @@ def test_production_mode_password_from_file(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_file_path=temp_file_path, + bundled_postgres=BundledPostgresConfig( + enabled=True, password_file_path=temp_file_path + ), ) assert config.password == "production_secret_password" @@ -87,26 +89,32 @@ def test_production_mode_password_from_file_with_whitespace(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_file_path=temp_file_path, + bundled_postgres=BundledPostgresConfig( + enabled=True, password_file_path=temp_file_path + ), ) assert config.password == "production_password_with_spaces" finally: os.unlink(temp_file_path) - def test_production_mode_password_file_not_found_raises_error(self): - """Test that non-existent password file raises ValueError.""" + def test_production_mode_password_file_not_found_returns_none(self): + """Test that non-existent password file raises ValueError in production.""" config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_file_path="/nonexistent/password/file", + bundled_postgres=BundledPostgresConfig( + password_file_path="/non/existent/path/to/password_file" + ), ) - with pytest.raises(FileNotFoundError): + with pytest.raises( + ValueError, match="Database password not provided in production mode" + ): _ = config.password def test_production_mode_password_file_permission_error(self): - """Test handling of file permission errors.""" + """Test handling of file permission errors raises ValueError in production.""" with tempfile.NamedTemporaryFile(delete=False) as temp_file: temp_file.write(b"secret") temp_file_path = temp_file.name @@ -118,10 +126,14 @@ def test_production_mode_password_file_permission_error(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_file_path=temp_file_path, + bundled_postgres=BundledPostgresConfig( + password_file_path=temp_file_path + ), ) - with pytest.raises(PermissionError): + with pytest.raises( + ValueError, match="Database password not provided in production mode" + ): _ = config.password finally: # Restore permissions for cleanup @@ -134,7 +146,9 @@ def test_production_mode_password_from_environment_variable(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_env_var="DB_PASSWORD", + bundled_postgres=BundledPostgresConfig( + enabled=True, password_env_var="DB_PASSWORD" + ), ) assert config.password == "env_secret_password" @@ -145,7 +159,9 @@ def test_production_mode_environment_variable_not_set_raises_error(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_env_var="MISSING_DB_PASSWORD", + bundled_postgres=BundledPostgresConfig( + enabled=True, password_env_var="MISSING_DB_PASSWORD" + ), ) with pytest.raises( @@ -159,7 +175,9 @@ def test_production_mode_empty_environment_variable_raises_error(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_env_var="DB_PASSWORD", + bundled_postgres=BundledPostgresConfig( + enabled=True, password_env_var="DB_PASSWORD" + ), ) with pytest.raises( @@ -178,8 +196,11 @@ def test_production_mode_env_var_takes_precedence_over_file(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_file_path=temp_file_path, - password_env_var="DB_PASSWORD", + bundled_postgres=BundledPostgresConfig( + enabled=True, + password_env_var="DB_PASSWORD", + password_file_path=temp_file_path, + ), ) assert config.password == "env_password" @@ -189,6 +210,7 @@ def test_production_mode_env_var_takes_precedence_over_file(self): def test_production_mode_no_password_source_raises_error(self): """Test that production mode without password sources raises ValueError.""" config = DatabaseConfig( + bundled_postgres=BundledPostgresConfig(enabled=True), url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", ) @@ -199,19 +221,6 @@ def test_production_mode_no_password_source_raises_error(self): ): _ = config.password - def test_invalid_environment_mode_raises_error(self): - """Test that invalid environment mode raises ValueError.""" - config = DatabaseConfig( - url="postgresql+asyncpg://user:password@localhost:5432/testdb", - environment_mode="invalid_mode", - ) - - with pytest.raises( - ValueError, - match="Invalid environment_mode; must be 'development', 'production', or 'test'", - ): - _ = config.password - def test_sqlite_url_in_development_mode(self): """Test password extraction from SQLite URL in development mode.""" config = DatabaseConfig( @@ -237,7 +246,9 @@ def test_production_mode_environment_variable_with_newlines(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_env_var="PROD_DB_PASS", + bundled_postgres=BundledPostgresConfig( + enabled=True, password_env_var="PROD_DB_PASS" + ), ) # Environment variables preserve newlines @@ -253,7 +264,9 @@ def test_production_mode_password_from_file_with_newlines(self): config = DatabaseConfig( url="postgresql+asyncpg://user@postgres:5432/appdb", environment_mode="production", - password_file_path=temp_file_path, + bundled_postgres=BundledPostgresConfig( + enabled=True, password_file_path=temp_file_path + ), ) # File reading strips whitespace including newlines @@ -290,3 +303,40 @@ def test_password_caching_behavior(self): password3 = config.password assert password1 == password2 == password3 == "testpass" + + def test_parsed_url_cache_cleared_on_url_change(self): + """Test that cached parsed_url is cleared when url field changes.""" + # Create config with initial URL + config = DatabaseConfig( + url="postgresql+asyncpg://user:password@localhost:5432/testdb", + environment_mode="development", + ) + + # Access parsed_url to populate the cache + initial_parsed = config.parsed_url + assert initial_parsed.host == "localhost" + assert initial_parsed.port == 5432 + assert initial_parsed.database == "testdb" + + # Verify cache exists + assert "parsed_url" in config.__dict__ + + # Create a new config with updated URL + # This triggers validation which should clear the cache + updated_config = DatabaseConfig( + url="postgresql+asyncpg://newuser:newpass@newhost:5433/newdb", + environment_mode="development", + ) + + # Access parsed_url on updated config - should reflect new URL + updated_parsed = updated_config.parsed_url + assert updated_parsed.host == "newhost" + assert updated_parsed.port == 5433 + assert updated_parsed.database == "newdb" + + # Verify the password also reflects the new URL + assert updated_config.password == "newpass" + + # Verify original config is unchanged (cache not shared between instances) + assert config.parsed_url.host == "localhost" + assert config.password == "password" diff --git a/tests/unit/cli/deployment/test_image_builder.py b/tests/unit/cli/deployment/test_image_builder.py index f1dfe0c..04c09da 100644 --- a/tests/unit/cli/deployment/test_image_builder.py +++ b/tests/unit/cli/deployment/test_image_builder.py @@ -8,8 +8,8 @@ import pytest -from src.cli.deployment.helm_deployer.constants import DeploymentConstants from src.cli.deployment.helm_deployer.image_builder import DeploymentError, ImageBuilder +from src.infra.constants import DeploymentConstants, DeploymentPaths class TestDeploymentError: @@ -68,15 +68,28 @@ def mock_console(self) -> MagicMock: """Create a mock Rich console.""" return MagicMock() + @pytest.fixture + def mock_controller(self) -> MagicMock: + """Create a mock Kubernetes controller.""" + controller = MagicMock() + controller.get_current_context = MagicMock() + controller.is_minikube_context = MagicMock() + return controller + @pytest.fixture def image_builder( - self, mock_commands: MagicMock, mock_console: MagicMock, tmp_path: Path + self, + mock_commands: MagicMock, + mock_console: MagicMock, + mock_controller: MagicMock, + tmp_path: Path, ) -> ImageBuilder: """Create an ImageBuilder instance with mocked dependencies.""" return ImageBuilder( commands=mock_commands, console=mock_console, - project_root=tmp_path, + controller=mock_controller, + paths=DeploymentPaths(project_root=tmp_path), constants=DeploymentConstants(), ) @@ -112,11 +125,11 @@ def test_cluster_type_detection_kind( def test_remote_cluster_without_registry_raises( self, image_builder: ImageBuilder, - mock_commands: MagicMock, + mock_controller: MagicMock, ) -> None: """Deploying to remote cluster without registry should raise error.""" - mock_commands.kubectl.is_minikube_context.return_value = False - mock_commands.kubectl.current_context.return_value = "gke-my-cluster" + mock_controller.is_minikube_context.return_value = False + mock_controller.get_current_context.return_value = "gke-my-cluster" with pytest.raises(DeploymentError) as exc_info: image_builder._load_images_to_cluster( @@ -131,10 +144,11 @@ def test_remote_cluster_without_registry_raises( def test_load_images_minikube( self, image_builder: ImageBuilder, + mock_controller: MagicMock, mock_commands: MagicMock, ) -> None: """Should load images into Minikube using minikube load.""" - mock_commands.kubectl.is_minikube_context.return_value = True + mock_controller.is_minikube_context.return_value = True mock_commands.docker.minikube_load_image.return_value = None image_builder._load_images_to_cluster( @@ -149,12 +163,12 @@ def test_load_images_minikube( def test_load_images_kind( self, image_builder: ImageBuilder, + mock_controller: MagicMock, mock_commands: MagicMock, ) -> None: """Should load images into Kind using kind load.""" - mock_commands.kubectl.is_minikube_context.return_value = False - # Need to mock get_current_context (note: without underscore) - mock_commands.kubectl.get_current_context.return_value = "kind-test" + mock_controller.is_minikube_context.return_value = False + mock_controller.get_current_context.return_value = "kind-test" mock_commands.docker.kind_load_image.return_value = None image_builder._load_images_to_cluster( @@ -169,11 +183,12 @@ def test_load_images_kind( def test_push_images_to_registry( self, image_builder: ImageBuilder, + mock_controller: MagicMock, mock_commands: MagicMock, ) -> None: """Should tag and push images when registry is provided.""" - mock_commands.kubectl.is_minikube_context.return_value = False - mock_commands.kubectl.current_context.return_value = "gke-prod" + mock_controller.is_minikube_context.return_value = False + mock_controller.get_current_context.return_value = "gke-prod" mock_commands.docker.tag_image.return_value = MagicMock(success=True) mock_commands.docker.push_image.return_value = MagicMock(success=True) @@ -247,7 +262,7 @@ def test_full_build_flow_for_minikube(self, full_project_setup: Path) -> None: builder = ImageBuilder( commands=mock_commands, console=mock_console, - project_root=full_project_setup, + paths=DeploymentPaths(project_root=full_project_setup), ) tag = builder.build_and_tag_images(MockProgress) # type: ignore[arg-type] diff --git a/tests/unit/cli/deployment/test_validator.py b/tests/unit/cli/deployment/test_validator.py index c2d2e7c..d551971 100644 --- a/tests/unit/cli/deployment/test_validator.py +++ b/tests/unit/cli/deployment/test_validator.py @@ -6,13 +6,13 @@ import pytest -from src.cli.deployment.helm_deployer.constants import DeploymentConstants from src.cli.deployment.helm_deployer.validator import ( DeploymentValidator, ValidationIssue, ValidationResult, ValidationSeverity, ) +from src.infra.constants import DeploymentConstants from src.infra.k8s.controller import JobInfo, PodInfo @@ -121,22 +121,38 @@ def mock_console(self) -> MagicMock: """Create a mock Rich console.""" return MagicMock() + @pytest.fixture + def mock_controller(self) -> MagicMock: + """Create a mock Kubernetes controller.""" + controller = MagicMock() + controller.namespace_exists = MagicMock() + controller.get_jobs = MagicMock() + controller.get_pods = MagicMock() + return controller + @pytest.fixture def validator( - self, mock_commands: MagicMock, mock_console: MagicMock + self, + mock_commands: MagicMock, + mock_console: MagicMock, + mock_controller: MagicMock, ) -> DeploymentValidator: """Create a validator instance with mocked dependencies.""" return DeploymentValidator( commands=mock_commands, console=mock_console, + controller=mock_controller, constants=DeploymentConstants(), ) def test_validate_fresh_namespace( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation of a non-existent namespace should return clean result.""" - mock_commands.kubectl.namespace_exists.return_value = False + mock_controller.namespace_exists.return_value = False result = validator.validate("api-forge-prod") @@ -145,13 +161,16 @@ def test_validate_fresh_namespace( assert len(result.issues) == 0 def test_validate_existing_namespace_no_issues( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation of existing namespace with no issues should be clean.""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [] - mock_commands.kubectl.get_pods.return_value = [] + mock_controller.get_jobs.return_value = [] + mock_controller.get_pods.return_value = [] result = validator.validate("api-forge-prod") @@ -159,19 +178,22 @@ def test_validate_existing_namespace_no_issues( assert result.namespace_exists is True def test_validate_detects_failed_jobs( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation should detect failed Kubernetes jobs. Init jobs like postgres-verifier are expected to have transient failures during startup, so they should be flagged as warnings, not errors. """ - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [ + mock_controller.get_jobs.return_value = [ JobInfo(name="postgres-verifier", status="Failed"), ] - mock_commands.kubectl.get_pods.return_value = [] + mock_controller.get_pods.return_value = [] result = validator.validate("api-forge-prod") @@ -183,15 +205,18 @@ def test_validate_detects_failed_jobs( assert "postgres-verifier" in result.issues[0].title def test_validate_any_failed_job_is_warning( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """All failed jobs should be flagged as warnings (may be transient).""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [ + mock_controller.get_jobs.return_value = [ JobInfo(name="migration-job", status="Failed"), ] - mock_commands.kubectl.get_pods.return_value = [] + mock_controller.get_pods.return_value = [] result = validator.validate("api-forge-prod") @@ -202,13 +227,16 @@ def test_validate_any_failed_job_is_warning( assert "migration-job" in result.issues[0].title def test_validate_detects_crashloop_pods( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation should detect pods in CrashLoopBackOff state.""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [] - mock_commands.kubectl.get_pods.return_value = [ + mock_controller.get_jobs.return_value = [] + mock_controller.get_pods.return_value = [ PodInfo(name="api-forge-app-xyz", status="CrashLoopBackOff"), ] @@ -221,13 +249,16 @@ def test_validate_detects_crashloop_pods( assert "CrashLoopBackOff" in result.issues[0].title def test_validate_detects_pending_pods( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation should detect pods stuck in Pending state.""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [] - mock_commands.kubectl.get_pods.return_value = [ + mock_controller.get_jobs.return_value = [] + mock_controller.get_pods.return_value = [ PodInfo(name="api-forge-app-xyz", status="Pending"), ] @@ -240,13 +271,16 @@ def test_validate_detects_pending_pods( assert "pending" in result.issues[0].title.lower() def test_validate_detects_error_pods( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation should detect pods in Error state.""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [] - mock_commands.kubectl.get_pods.return_value = [ + mock_controller.get_jobs.return_value = [] + mock_controller.get_pods.return_value = [ PodInfo(name="api-forge-app-xyz", status="Error"), ] @@ -258,17 +292,20 @@ def test_validate_detects_error_pods( assert "Error" in result.issues[0].title def test_validate_job_pods_only_checks_most_recent( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """For job-owned pods, only the most recent pod should be checked. If old pods from a job are in Error state but a newer pod succeeded, we should not flag the old errors. """ - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [] - mock_commands.kubectl.get_pods.return_value = [ + mock_controller.get_jobs.return_value = [] + mock_controller.get_pods.return_value = [ # Old pod from first attempt - failed PodInfo( name="postgres-verifier-abc", @@ -292,13 +329,16 @@ def test_validate_job_pods_only_checks_most_recent( assert len(result.issues) == 0 def test_validate_job_pods_flags_if_most_recent_failed( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """If the most recent job pod is in Error state, flag it as a warning.""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [] - mock_commands.kubectl.get_pods.return_value = [ + mock_controller.get_jobs.return_value = [] + mock_controller.get_pods.return_value = [ # Old pod succeeded PodInfo( name="postgres-verifier-abc", @@ -324,15 +364,18 @@ def test_validate_job_pods_flags_if_most_recent_failed( assert "postgres-verifier-def" in result.issues[0].title def test_validate_detects_multiple_issues( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """Validation should accumulate multiple issues.""" - mock_commands.kubectl.namespace_exists.return_value = True + mock_controller.namespace_exists.return_value = True mock_commands.helm.list_releases.return_value = [] - mock_commands.kubectl.get_jobs.return_value = [ + mock_controller.get_jobs.return_value = [ JobInfo(name="postgres-verifier", status="Failed"), ] - mock_commands.kubectl.get_pods.return_value = [ + mock_controller.get_pods.return_value = [ PodInfo(name="api-forge-app-xyz", status="CrashLoopBackOff"), PodInfo(name="api-forge-worker-abc", status="Pending"), ] @@ -426,19 +469,22 @@ def test_prompt_cleanup_returns_false_on_no( assert should_cleanup is False def test_run_cleanup_uninstalls_helm_and_deletes_resources( - self, validator: DeploymentValidator, mock_commands: MagicMock + self, + validator: DeploymentValidator, + mock_controller: MagicMock, + mock_commands: MagicMock, ) -> None: """run_cleanup should uninstall Helm release and delete PVCs/namespace.""" mock_commands.helm.uninstall.return_value = MagicMock(success=True) - mock_commands.kubectl.delete_pvcs.return_value = MagicMock(success=True) - mock_commands.kubectl.delete_namespace.return_value = MagicMock(success=True) + mock_controller.delete_pvcs.return_value = MagicMock(success=True) + mock_controller.delete_namespace.return_value = MagicMock(success=True) result = validator.run_cleanup("api-forge-prod") assert result is True mock_commands.helm.uninstall.assert_called_once() - mock_commands.kubectl.delete_pvcs.assert_called_once_with("api-forge-prod") - mock_commands.kubectl.delete_namespace.assert_called_once() + mock_controller.delete_pvcs.assert_called_once_with("api-forge-prod") + mock_controller.delete_namespace.assert_called_once() def test_run_cleanup_handles_failure( self, @@ -452,6 +498,7 @@ def test_run_cleanup_handles_failure( result = validator.run_cleanup("api-forge-prod") assert result is False - # Should print error message - call_args = str(mock_console.print.call_args_list) + # Should call error method with failure message + mock_console.error.assert_called_once() + call_args = str(mock_console.error.call_args_list) assert "failed" in call_args.lower() or "error" in call_args.lower() diff --git a/tests/unit/cli/test_cli_context.py b/tests/unit/cli/test_cli_context.py new file mode 100644 index 0000000..2902abe --- /dev/null +++ b/tests/unit/cli/test_cli_context.py @@ -0,0 +1,173 @@ +"""Tests for CLI context dependency injection.""" + +from pathlib import Path +from unittest.mock import Mock, patch + +import click +import pytest +import typer + +from src.cli.context import CLIContext, build_cli_context, get_cli_context + + +def test_cli_context_is_immutable(): + """Test that CLIContext is frozen/immutable.""" + ctx = CLIContext( + console=Mock(), + project_root=Path("/test"), + commands=Mock(), + k8s_controller=Mock(), + constants=Mock(), + paths=Mock(), + ) + + with pytest.raises(AttributeError): + ctx.console = Mock() # type: ignore[attr-defined] + + +@patch("src.cli.context.get_project_root") +@patch("src.cli.context.get_k8s_controller_sync") +def test_build_cli_context_creates_all_dependencies(mock_k8s_controller, mock_get_root): + """Test that build_cli_context creates all required dependencies.""" + mock_get_root.return_value = Path("/test/project") + mock_k8s_controller.return_value = Mock() + + ctx = build_cli_context() + + assert ctx.console is not None + assert ctx.project_root == Path("/test/project") + assert ctx.commands is not None + assert ctx.k8s_controller is not None + assert ctx.constants is not None + assert ctx.paths is not None + + +@patch("src.cli.context.get_project_root") +def test_build_cli_context_paths_uses_project_root(mock_get_root): + """Test that DeploymentPaths is initialized with project root.""" + mock_get_root.return_value = Path("/test/project") + + ctx = build_cli_context() + + # Paths should have project_root as base + assert hasattr(ctx.paths, "project_root") + + +def test_get_cli_context_from_typer_context(): + """Test that get_cli_context retrieves from Typer context.""" + mock_ctx_obj = CLIContext( + console=Mock(), + project_root=Path("/test"), + commands=Mock(), + k8s_controller=Mock(), + constants=Mock(), + paths=Mock(), + ) + + typer_ctx = Mock(spec=typer.Context) + typer_ctx.obj = mock_ctx_obj + + result = get_cli_context(typer_ctx) + + assert result is mock_ctx_obj + + +def test_get_cli_context_with_none_falls_back(): + """Test that get_cli_context creates new context when ctx is None.""" + with patch("src.cli.context.build_cli_context") as mock_build: + mock_build.return_value = Mock(spec=CLIContext) + + get_cli_context(None) + + # Should have called build_cli_context + mock_build.assert_called_once() + + +def test_get_cli_context_with_invalid_obj_falls_back(): + """Test that get_cli_context falls back when ctx.obj is not CLIContext.""" + typer_ctx = Mock(spec=typer.Context) + typer_ctx.obj = "invalid" # Not a CLIContext + + with patch("src.cli.context.build_cli_context") as mock_build: + mock_build.return_value = Mock(spec=CLIContext) + + get_cli_context(typer_ctx) + + # Should have called build_cli_context + mock_build.assert_called_once() + + +@patch("click.get_current_context") +def test_get_cli_context_uses_click_context_as_fallback(mock_get_click_ctx): + """Test that get_cli_context uses click context when typer ctx is None.""" + mock_ctx_obj = CLIContext( + console=Mock(), + project_root=Path("/test"), + commands=Mock(), + k8s_controller=Mock(), + constants=Mock(), + paths=Mock(), + ) + + mock_click_context = Mock() + mock_click_context.obj = mock_ctx_obj + mock_get_click_ctx.return_value = mock_click_context + + result = get_cli_context(None) + + assert result is mock_ctx_obj + mock_get_click_ctx.assert_called_once_with(silent=True) + + +@patch("src.cli.context.ShellCommands") +@patch("src.cli.context.get_project_root") +def test_cli_context_shell_commands_initialized_with_project_root( + mock_get_root, mock_shell_commands +): + """Test that ShellCommands is initialized with project_root.""" + mock_get_root.return_value = Path("/test/project") + + build_cli_context() + + mock_shell_commands.assert_called_once_with(Path("/test/project")) + + +def test_cli_context_all_fields_accessible(): + """Test that all CLIContext fields are accessible.""" + ctx = CLIContext( + console=Mock(), + project_root=Path("/test"), + commands=Mock(), + k8s_controller=Mock(), + constants=Mock(), + paths=Mock(), + ) + + # All fields should be accessible + assert ctx.console is not None + assert ctx.project_root == Path("/test") + assert ctx.commands is not None + assert ctx.k8s_controller is not None + assert ctx.constants is not None + assert ctx.paths is not None + + +@patch("src.cli.context.DeploymentConstants") +@patch("src.cli.context.DeploymentPaths") +@patch("src.cli.context.get_project_root") +def test_cli_context_constants_and_paths_initialized( + mock_get_root, mock_paths_cls, mock_constants_cls +): + """Test that DeploymentConstants and DeploymentPaths are initialized.""" + mock_get_root.return_value = Path("/test/project") + mock_constants = Mock() + mock_paths = Mock() + mock_constants_cls.return_value = mock_constants + mock_paths_cls.return_value = mock_paths + + ctx = build_cli_context() + + assert ctx.constants is mock_constants + assert ctx.paths is mock_paths + mock_constants_cls.assert_called_once_with() + mock_paths_cls.assert_called_once_with(Path("/test/project")) diff --git a/tests/unit/cli/test_compose_runner.py b/tests/unit/cli/test_compose_runner.py new file mode 100644 index 0000000..6980077 --- /dev/null +++ b/tests/unit/cli/test_compose_runner.py @@ -0,0 +1,290 @@ +"""Tests for ComposeRunner helper.""" + +import subprocess +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from src.cli.shared.compose import ComposeRunner + + +@pytest.fixture +def compose_runner(): + """Create a ComposeRunner instance for testing.""" + return ComposeRunner( + Path("/test/project"), + compose_file=Path("/test/project/docker-compose.test.yml"), + project_name="test-project", + ) + + +@pytest.fixture +def compose_runner_no_project(): + """Create a ComposeRunner without project name.""" + return ComposeRunner( + Path("/test/project"), + compose_file=Path("/test/project/docker-compose.yml"), + ) + + +def test_base_cmd_with_project_name(compose_runner): + """Test that base command includes project name when provided.""" + cmd = compose_runner._base_cmd() + + assert cmd == [ + "docker", + "compose", + "-p", + "test-project", + "-f", + "/test/project/docker-compose.test.yml", + ] + + +def test_base_cmd_without_project_name(compose_runner_no_project): + """Test that base command works without project name.""" + cmd = compose_runner_no_project._base_cmd() + + assert cmd == [ + "docker", + "compose", + "-f", + "/test/project/docker-compose.yml", + ] + + +@patch("subprocess.run") +def test_run_executes_command(mock_run, compose_runner): + """Test that run() executes subprocess with correct arguments.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.run(["up", "-d"]) + + mock_run.assert_called_once() + call_args = mock_run.call_args + + expected_cmd = [ + "docker", + "compose", + "-p", + "test-project", + "-f", + "/test/project/docker-compose.test.yml", + "up", + "-d", + ] + + assert call_args.args[0] == expected_cmd + assert call_args.kwargs["cwd"] == Path("/test/project") + assert call_args.kwargs["text"] is True + + +@patch("subprocess.run") +def test_run_with_capture_output(mock_run, compose_runner): + """Test that run() respects capture_output flag.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="output", stderr="" + ) + + compose_runner.run(["ps"], capture_output=True) + + assert mock_run.call_args.kwargs["capture_output"] is True + + +@patch("subprocess.run") +def test_run_with_check(mock_run, compose_runner): + """Test that run() respects check flag.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.run(["up"], check=True) + + assert mock_run.call_args.kwargs["check"] is True + + +@patch("subprocess.run") +def test_logs_without_service(mock_run, compose_runner): + """Test logs() without specific service.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.logs() + + expected_cmd = [ + "docker", + "compose", + "-p", + "test-project", + "-f", + "/test/project/docker-compose.test.yml", + "logs", + ] + + assert mock_run.call_args.args[0] == expected_cmd + + +@patch("subprocess.run") +def test_logs_with_service(mock_run, compose_runner): + """Test logs() with specific service.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.logs(service="postgres") + + cmd = mock_run.call_args.args[0] + assert cmd[-1] == "postgres" + + +@patch("subprocess.run") +def test_logs_with_follow(mock_run, compose_runner): + """Test logs() with follow flag.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.logs(follow=True) + + cmd = mock_run.call_args.args[0] + assert "--follow" in cmd + + +@patch("subprocess.run") +def test_logs_with_tail(mock_run, compose_runner): + """Test logs() with tail option.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.logs(tail=50) + + cmd = mock_run.call_args.args[0] + assert "--tail=50" in cmd + + +@patch("subprocess.run") +def test_logs_with_all_options(mock_run, compose_runner): + """Test logs() with all options combined.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.logs(service="app", follow=True, tail=100) + + cmd = mock_run.call_args.args[0] + assert "--tail=100" in cmd + assert "--follow" in cmd + assert cmd[-1] == "app" + + +@patch("subprocess.run") +def test_restart_without_service(mock_run, compose_runner): + """Test restart() without specific service.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.restart() + + expected_cmd = [ + "docker", + "compose", + "-p", + "test-project", + "-f", + "/test/project/docker-compose.test.yml", + "restart", + ] + + assert mock_run.call_args.args[0] == expected_cmd + + +@patch("subprocess.run") +def test_restart_with_service(mock_run, compose_runner): + """Test restart() with specific service.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.restart(service="redis") + + cmd = mock_run.call_args.args[0] + assert cmd[-1] == "redis" + + +@patch("subprocess.run") +def test_build_without_service(mock_run, compose_runner): + """Test build() without specific service.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.build() + + expected_cmd = [ + "docker", + "compose", + "-p", + "test-project", + "-f", + "/test/project/docker-compose.test.yml", + "build", + ] + + assert mock_run.call_args.args[0] == expected_cmd + + +@patch("subprocess.run") +def test_build_with_service(mock_run, compose_runner): + """Test build() with specific service.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.build(service="app") + + cmd = mock_run.call_args.args[0] + assert cmd[-1] == "app" + + +@patch("subprocess.run") +def test_build_with_no_cache(mock_run, compose_runner): + """Test build() with no_cache flag.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.build(no_cache=True) + + cmd = mock_run.call_args.args[0] + assert "--no-cache" in cmd + + +@patch("subprocess.run") +def test_build_with_service_and_no_cache(mock_run, compose_runner): + """Test build() with both service and no_cache.""" + mock_run.return_value = subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + + compose_runner.build(service="app", no_cache=True) + + cmd = mock_run.call_args.args[0] + assert "--no-cache" in cmd + assert cmd[-1] == "app" + + +@patch("subprocess.run") +def test_check_flag_propagates_exceptions(mock_run, compose_runner): + """Test that check=True causes CalledProcessError to be raised.""" + mock_run.side_effect = subprocess.CalledProcessError( + returncode=1, cmd=["docker", "compose", "up"] + ) + + with pytest.raises(subprocess.CalledProcessError): + compose_runner.run(["up"], check=True) diff --git a/tests/unit/cli/test_console_error_handling.py b/tests/unit/cli/test_console_error_handling.py new file mode 100644 index 0000000..959d327 --- /dev/null +++ b/tests/unit/cli/test_console_error_handling.py @@ -0,0 +1,27 @@ +import pytest +import typer + +from src.cli.deployment.helm_deployer.image_builder import DeploymentError +from src.cli.shared.console import with_error_handling + + +def test_with_error_handling_handles_deployment_error(): + @with_error_handling + def _command() -> None: + raise DeploymentError("Boom", details="extra") + + with pytest.raises(typer.Exit) as excinfo: + _command() + + assert excinfo.value.exit_code == 1 + + +def test_with_error_handling_handles_keyboard_interrupt(): + @with_error_handling + def _command() -> None: + raise KeyboardInterrupt + + with pytest.raises(typer.Exit) as excinfo: + _command() + + assert excinfo.value.exit_code == 130 diff --git a/tests/unit/cli/test_db_runtime.py b/tests/unit/cli/test_db_runtime.py new file mode 100644 index 0000000..f730df7 --- /dev/null +++ b/tests/unit/cli/test_db_runtime.py @@ -0,0 +1,354 @@ +"""Tests for database runtime adapters.""" + +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from src.cli.commands.db.runtime import DbRuntime, no_port_forward + + +def test_no_port_forward_is_noop_context_manager(): + """Test that no_port_forward returns a no-op context manager.""" + with no_port_forward() as result: + assert result is None + + +def test_db_runtime_is_immutable(): + """Test that DbRuntime is frozen/immutable.""" + runtime = DbRuntime( + name="test", + console=Mock(), + get_settings=Mock(), + connect=Mock(), + port_forward=Mock(), + get_deployer=Mock(), + secrets_dirs=[Path("/tmp")], + is_temporal_enabled=Mock(), + is_bundled_postgres_enabled=Mock(), + ) + + with pytest.raises(AttributeError): + runtime.name = "changed" # type: ignore[attr-defined] + + +@patch("src.cli.commands.db.runtime_compose.get_settings") +@patch("src.cli.commands.db.runtime_compose.get_docker_compose_postgres_connection") +def test_compose_runtime_factory(mock_get_conn, mock_get_settings): + """Test that get_compose_runtime returns a properly configured DbRuntime.""" + from src.cli.commands.db.runtime_compose import get_compose_runtime + + runtime = get_compose_runtime() + + assert runtime.name == "compose" + assert runtime.console is not None + assert callable(runtime.get_settings) + assert callable(runtime.connect) + assert callable(runtime.port_forward) + assert callable(runtime.get_deployer) + assert len(runtime.secrets_dirs) > 0 + assert callable(runtime.is_temporal_enabled) + assert callable(runtime.is_bundled_postgres_enabled) + + +@patch("src.cli.commands.db.runtime_k8s.get_settings") +@patch("src.cli.commands.db.runtime_k8s.get_k8s_postgres_connection") +@patch("src.cli.commands.db.runtime_k8s.postgres_port_forward_if_needed") +def test_k8s_runtime_factory(mock_port_forward, mock_get_conn, mock_get_settings): + """Test that get_k8s_runtime returns a properly configured DbRuntime.""" + from src.cli.commands.db.runtime_k8s import get_k8s_runtime + + runtime = get_k8s_runtime() + + assert runtime.name == "k8s" + assert runtime.console is not None + assert callable(runtime.get_settings) + assert callable(runtime.connect) + assert callable(runtime.port_forward) + assert callable(runtime.get_deployer) + assert len(runtime.secrets_dirs) > 0 + assert callable(runtime.is_temporal_enabled) + assert callable(runtime.is_bundled_postgres_enabled) + + +def test_compose_runtime_port_forward_returns_nullcontext(): + """Test that compose runtime port_forward returns a no-op context.""" + from src.cli.commands.db.runtime_compose import get_compose_runtime + + runtime = get_compose_runtime() + + # Should return a context manager that does nothing + with runtime.port_forward() as result: + assert result is None + + +@patch("src.cli.commands.db.runtime_k8s.postgres_port_forward_if_needed") +@patch("src.cli.commands.db.runtime_k8s.get_namespace") +@patch("src.cli.commands.db.runtime_k8s.get_postgres_label") +def test_k8s_runtime_port_forward_uses_namespace_and_label( + mock_get_label, mock_get_ns, mock_port_forward +): + """Test that k8s runtime port_forward uses proper namespace and label.""" + from src.cli.commands.db.runtime_k8s import get_k8s_runtime + + mock_get_ns.return_value = "test-namespace" + mock_get_label.return_value = "app=postgres" + mock_port_forward.return_value = MagicMock() + + runtime = get_k8s_runtime() + runtime.port_forward() + + # Verify port_forward was called with correct params + mock_port_forward.assert_called_once_with( + namespace="test-namespace", pod_label="app=postgres" + ) + + +def test_runtime_connect_callable_signature(): + """Test that runtime connect callable has expected signature.""" + from src.cli.commands.db.runtime_compose import get_compose_runtime + + runtime = get_compose_runtime() + + # Should accept settings and superuser_mode + mock_settings = Mock() + mock_settings.host = "localhost" + + # This will fail if signature is wrong + with patch( + "src.cli.commands.db.runtime_compose.get_docker_compose_postgres_connection" + ) as mock_conn: + mock_conn.return_value = Mock() + runtime.connect(mock_settings, True) + mock_conn.assert_called_once_with(mock_settings, superuser_mode=True) + + mock_conn.reset_mock() + runtime.connect(mock_settings, False) + mock_conn.assert_called_once_with(mock_settings, superuser_mode=False) + + +# ============================================================================= +# Workflow Tests +# ============================================================================= + + +@pytest.fixture +def mock_runtime(): + """Create a mock DbRuntime for testing workflows.""" + runtime = MagicMock(spec=DbRuntime) + runtime.name = "test" + runtime.console = Mock() + runtime.get_settings = Mock() + runtime.connect = MagicMock() + runtime.port_forward = no_port_forward + runtime.get_deployer = Mock() + runtime.secrets_dirs = [Path("/fake/secrets")] + runtime.is_temporal_enabled = Mock(return_value=False) + runtime.is_bundled_postgres_enabled = Mock(return_value=False) + return runtime + + +class TestRunInit: + """Test run_init workflow.""" + + @patch("src.infra.postgres.PostgresInitializer") + def test_run_init_success(self, mock_initializer_class, mock_runtime): + """Verify run_init() calls PostgresInitializer.initialize().""" + from src.cli.commands.db import run_init + + # Setup mocks + mock_settings = Mock() + mock_settings.ensure_all_passwords.return_value = mock_settings + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_initializer = Mock() + mock_initializer.initialize.return_value = True + mock_initializer_class.return_value = mock_initializer + + # Execute + result = run_init(mock_runtime) + + # Verify + assert result is True + mock_runtime.get_settings.assert_called_once() + mock_settings.ensure_all_passwords.assert_called_once() + mock_runtime.connect.assert_called_once_with(mock_settings, True) + mock_initializer_class.assert_called_once_with(connection=mock_conn) + mock_initializer.initialize.assert_called_once() + + @patch("src.infra.postgres.PostgresInitializer") + def test_run_init_failure(self, mock_initializer_class, mock_runtime): + """Verify run_init() returns False when initialization fails.""" + from src.cli.commands.db import run_init + + mock_settings = Mock() + mock_settings.ensure_all_passwords.return_value = mock_settings + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_initializer = Mock() + mock_initializer.initialize.return_value = False + mock_initializer_class.return_value = mock_initializer + + result = run_init(mock_runtime) + + assert result is False + + +class TestRunVerify: + """Test run_verify workflow.""" + + @patch("src.infra.postgres.PostgresVerifier") + def test_run_verify_with_superuser_mode(self, mock_verifier_class, mock_runtime): + """Verify run_verify() uses superuser_mode parameter correctly.""" + from src.cli.commands.db import run_verify + + mock_settings = Mock() + mock_settings.ensure_all_passwords.return_value = mock_settings + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_verifier = Mock() + mock_verifier.verify_all.return_value = True + mock_verifier_class.return_value = mock_verifier + + result = run_verify(mock_runtime, superuser_mode=True) + + assert result is True + mock_runtime.connect.assert_called_once_with(mock_settings, True) + + @patch("src.infra.postgres.PostgresVerifier") + def test_run_verify_without_superuser_mode(self, mock_verifier_class, mock_runtime): + """Verify run_verify() can run without superuser mode.""" + from src.cli.commands.db import run_verify + + mock_settings = Mock() + mock_settings.ensure_all_passwords.return_value = mock_settings + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_verifier = Mock() + mock_verifier.verify_all.return_value = True + mock_verifier_class.return_value = mock_verifier + + result = run_verify(mock_runtime, superuser_mode=False) + + assert result is True + mock_runtime.connect.assert_called_once_with(mock_settings, False) + + +class TestRunBackup: + """Test run_backup workflow.""" + + @patch("src.infra.postgres.PostgresBackup") + def test_run_backup_returns_tuple(self, mock_backup_class, mock_runtime): + """Verify run_backup() returns (success, path) tuple.""" + from src.cli.commands.db import run_backup + + mock_settings = Mock() + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_backup = Mock() + mock_backup.create_backup.return_value = (True, "/path/to/backup.sql") + mock_backup_class.return_value = mock_backup + + output_dir = Path("/tmp/backups") + success, path = run_backup( + mock_runtime, output_dir=output_dir, superuser_mode=False + ) + + assert success is True + assert path == "/path/to/backup.sql" + mock_backup_class.assert_called_once_with( + connection=mock_conn, backup_dir=output_dir + ) + + +class TestRunReset: + """Test run_reset workflow.""" + + @patch("src.infra.postgres.PostgresReset") + def test_run_reset_with_temporal(self, mock_reset_class, mock_runtime): + """Verify run_reset() passes include_temporal parameter.""" + from src.cli.commands.db import run_reset + + mock_settings = Mock() + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_reset = Mock() + mock_reset.reset.return_value = True + mock_reset_class.return_value = mock_reset + + result = run_reset(mock_runtime, include_temporal=True, superuser_mode=True) + + assert result is True + mock_reset.reset.assert_called_once_with(include_temporal=True) + + +class TestRunSync: + """Test run_sync workflow.""" + + @patch("src.infra.postgres.PostgresPasswordSync") + def test_run_sync_without_bundled_postgres(self, mock_sync_class, mock_runtime): + """Verify run_sync() skips bundled sync when not enabled.""" + from src.cli.commands.db import run_sync + + mock_runtime.is_bundled_postgres_enabled.return_value = False + + mock_settings = Mock() + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_sync = Mock() + mock_sync.sync_user_roles_and_passwords.return_value = True + mock_sync_class.return_value = mock_sync + + result = run_sync(mock_runtime) + + assert result is True + # Should only call get_settings once (at start of function) + assert mock_runtime.get_settings.call_count == 1 + + @patch("src.infra.postgres.PostgresPasswordSync") + def test_run_sync_with_bundled_postgres(self, mock_sync_class, mock_runtime): + """Verify run_sync() handles bundled postgres sync.""" + from src.cli.commands.db import run_sync + + mock_runtime.is_bundled_postgres_enabled.return_value = True + + mock_settings = Mock() + mock_runtime.get_settings.return_value = mock_settings + + mock_conn = MagicMock() + mock_runtime.connect.return_value.__enter__.return_value = mock_conn + + mock_deployer = Mock() + mock_runtime.get_deployer.return_value = mock_deployer + + mock_sync = Mock() + mock_sync.sync_bundled_superuser_password.return_value = True + mock_sync.sync_user_roles_and_passwords.return_value = True + mock_sync_class.return_value = mock_sync + + result = run_sync(mock_runtime) + + assert result is True + # Should only call get_settings once (at start of function) + assert mock_runtime.get_settings.call_count == 1 diff --git a/tests/unit/cli/test_db_utils.py b/tests/unit/cli/test_db_utils.py new file mode 100644 index 0000000..b39b9e4 --- /dev/null +++ b/tests/unit/cli/test_db_utils.py @@ -0,0 +1,65 @@ +import pytest +import typer + +from src.cli.commands import db_utils + + +def test_parse_connection_string_basic(): + conn_str = "postgres://user:pass@db.example.com:5432/mydb?sslmode=require" + parsed = db_utils.parse_connection_string(conn_str) + + assert parsed["username"] == "user" + assert parsed["password"] == "pass" + assert parsed["host"] == "db.example.com" + assert parsed["port"] == "5432" + assert parsed["database"] == "mydb" + assert parsed["sslmode"] == "require" + + +def test_build_connection_string_includes_password_when_requested(): + conn = db_utils.build_connection_string( + username="app", + host="localhost", + port="5432", + database="appdb", + sslmode="verify-full", + include_password=True, + password="secret", + ) + + assert conn == "postgres://app:secret@localhost:5432/appdb?sslmode=verify-full" + + +def test_validate_external_db_params_applies_overrides(): + conn_str = "postgres://user:pass@db.example.com:5432/mydb?sslmode=require" + result = db_utils.validate_external_db_params( + connection_string=conn_str, + host="override.example.com", + port=None, + username="override_user", + password="override_pass", + database=None, + sslmode="verify-full", + ) + + assert result == ( + "override.example.com", + "5432", + "override_user", + "override_pass", + "mydb", + "verify-full", + ) + + +def test_validate_external_db_params_missing_required_fields_raises(): + with pytest.raises(typer.Exit): + db_utils.validate_external_db_params( + connection_string=None, + host=None, + port=None, + username=None, + password=None, + database=None, + sslmode=None, + ) diff --git a/tests/unit/cli/test_db_workflows.py b/tests/unit/cli/test_db_workflows.py new file mode 100644 index 0000000..98f9bba --- /dev/null +++ b/tests/unit/cli/test_db_workflows.py @@ -0,0 +1,349 @@ +"""Tests for database workflow functions.""" + +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +import pytest +import typer + +from src.cli.commands.db.runtime import DbRuntime, no_port_forward +from src.cli.commands.db.workflows import ( + run_backup, + run_init, + run_migrate, + run_reset, + run_status, + run_sync, + run_verify, +) + + +@pytest.fixture +def mock_runtime(): + """Create a mock DbRuntime for testing.""" + runtime = Mock(spec=DbRuntime) + runtime.name = "test" + runtime.console = Mock() + runtime.get_settings = Mock() + runtime.connect = Mock() + runtime.port_forward = Mock(return_value=no_port_forward()) + runtime.get_deployer = Mock() + runtime.secrets_dirs = [Path("/tmp")] + runtime.is_temporal_enabled = Mock(return_value=False) + runtime.is_bundled_postgres_enabled = Mock(return_value=False) + return runtime + + +@pytest.fixture +def mock_connection(): + """Create a mock PostgresConnection.""" + conn = MagicMock() + conn.__enter__ = Mock(return_value=conn) + conn.__exit__ = Mock(return_value=False) + conn.scalar = Mock() + conn.get_connection_string = Mock(return_value="postgres://test") + return conn + + +@pytest.fixture +def mock_settings(): + """Create mock database settings.""" + settings = Mock() + settings.ensure_all_passwords = Mock(return_value=settings) + settings.ensure_superuser_password = Mock(return_value=settings) + settings.host = "localhost" + settings.port = 5432 + settings.app_db = "appdb" + return settings + + +def test_run_init_success(mock_runtime, mock_connection, mock_settings): + """Test successful database initialization.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresInitializer") as mock_init: + mock_initializer = Mock() + mock_initializer.initialize.return_value = True + mock_init.return_value = mock_initializer + + result = run_init(mock_runtime) + + assert result is True + mock_init.assert_called_once_with(connection=mock_connection) + mock_initializer.initialize.assert_called_once() + + +def test_run_init_failure(mock_runtime, mock_connection, mock_settings): + """Test failed database initialization.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresInitializer") as mock_init: + mock_initializer = Mock() + mock_initializer.initialize.return_value = False + mock_init.return_value = mock_initializer + + result = run_init(mock_runtime) + + assert result is False + + +def test_run_verify_with_superuser(mock_runtime, mock_connection, mock_settings): + """Test database verification with superuser mode.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresVerifier") as mock_verifier: + mock_verifier_instance = Mock() + mock_verifier_instance.verify_all.return_value = True + mock_verifier.return_value = mock_verifier_instance + + result = run_verify(mock_runtime, superuser_mode=True) + + assert result is True + mock_runtime.connect.assert_called_once_with(mock_settings, True) + + +def test_run_verify_without_superuser(mock_runtime, mock_connection, mock_settings): + """Test database verification without superuser mode.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresVerifier") as mock_verifier: + mock_verifier_instance = Mock() + mock_verifier_instance.verify_all.return_value = True + mock_verifier.return_value = mock_verifier_instance + + result = run_verify(mock_runtime, superuser_mode=False) + + assert result is True + mock_runtime.connect.assert_called_once_with(mock_settings, False) + + +def test_run_sync_with_bundled_postgres(mock_runtime, mock_connection, mock_settings): + """Test password sync when bundled postgres is enabled.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + mock_runtime.is_bundled_postgres_enabled.return_value = True + + with patch("src.infra.postgres.PostgresPasswordSync") as mock_sync: + mock_sync_instance = Mock() + mock_sync_instance.sync_bundled_superuser_password.return_value = True + mock_sync_instance.sync_user_roles_and_passwords.return_value = True + mock_sync.return_value = mock_sync_instance + + result = run_sync(mock_runtime) + + assert result is True + # Should be called twice - once for bundled, once for users + assert mock_sync.call_count == 2 + mock_sync_instance.sync_bundled_superuser_password.assert_called_once() + mock_sync_instance.sync_user_roles_and_passwords.assert_called_once() + + +def test_run_sync_without_bundled_postgres( + mock_runtime, mock_connection, mock_settings +): + """Test password sync when bundled postgres is disabled.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + mock_runtime.is_bundled_postgres_enabled.return_value = False + + with patch("src.infra.postgres.PostgresPasswordSync") as mock_sync: + mock_sync_instance = Mock() + mock_sync_instance.sync_user_roles_and_passwords.return_value = True + mock_sync.return_value = mock_sync_instance + + result = run_sync(mock_runtime) + + assert result is True + # Should only be called once for users + mock_sync.assert_called_once() + mock_sync_instance.sync_user_roles_and_passwords.assert_called_once() + + +def test_run_backup_success(mock_runtime, mock_connection, mock_settings): + """Test successful database backup.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresBackup") as mock_backup: + mock_backup_instance = Mock() + mock_backup_instance.create_backup.return_value = (True, "/path/to/backup.sql") + mock_backup.return_value = mock_backup_instance + + output_dir = Path("/tmp/backups") + success, result = run_backup( + mock_runtime, output_dir=output_dir, superuser_mode=True + ) + + assert success is True + assert result == "/path/to/backup.sql" + mock_backup.assert_called_once_with( + connection=mock_connection, backup_dir=output_dir + ) + + +def test_run_backup_failure(mock_runtime, mock_connection, mock_settings): + """Test failed database backup.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresBackup") as mock_backup: + mock_backup_instance = Mock() + mock_backup_instance.create_backup.return_value = (False, "Backup failed") + mock_backup.return_value = mock_backup_instance + + success, result = run_backup( + mock_runtime, output_dir=Path("/tmp"), superuser_mode=False + ) + + assert success is False + assert result == "Backup failed" + + +def test_run_reset_with_temporal(mock_runtime, mock_connection, mock_settings): + """Test database reset including temporal.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresReset") as mock_reset: + mock_reset_instance = Mock() + mock_reset_instance.reset.return_value = True + mock_reset.return_value = mock_reset_instance + + result = run_reset(mock_runtime, include_temporal=True, superuser_mode=True) + + assert result is True + mock_reset_instance.reset.assert_called_once_with(include_temporal=True) + + +def test_run_reset_without_temporal(mock_runtime, mock_connection, mock_settings): + """Test database reset excluding temporal.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.infra.postgres.PostgresReset") as mock_reset: + mock_reset_instance = Mock() + mock_reset_instance.reset.return_value = True + mock_reset.return_value = mock_reset_instance + + result = run_reset(mock_runtime, include_temporal=False, superuser_mode=False) + + assert result is True + mock_reset_instance.reset.assert_called_once_with(include_temporal=False) + + +def test_run_status_displays_metrics(mock_runtime, mock_connection, mock_settings): + """Test that status command displays database metrics.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + mock_runtime.is_temporal_enabled.return_value = False + + # Mock scalar queries + mock_connection.scalar.side_effect = [ + 3600.0, # uptime + 5, # active connections + 10, # total connections + 100, # max connections + 95.5, # cache hit ratio + "100 MB", # database size + 5, # table count + 1000, # row count + ] + + run_status(mock_runtime, superuser_mode=True) + + # Verify output was printed + assert mock_runtime.console.print.called + + +def test_run_status_handles_connection_error(mock_runtime, mock_settings): + """Test that status command handles connection errors gracefully.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.side_effect = Exception("Connection failed") + + # Should not raise, should handle gracefully + run_status(mock_runtime, superuser_mode=False) + + mock_runtime.console.error.assert_called() + + +def test_run_migrate_success(mock_runtime, mock_connection, mock_settings): + """Test successful migration execution.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.cli.commands.db.workflows.run_migration") as mock_migrate: + mock_migrate.return_value = True + + # Should not raise + run_migrate( + mock_runtime, + action="upgrade", + revision="head", + message=None, + merge_revisions=[], + purge=False, + autogenerate=False, + sql=False, + ) + + mock_migrate.assert_called_once() + call_kwargs = mock_migrate.call_args.kwargs + assert call_kwargs["action"] == "upgrade" + assert call_kwargs["revision"] == "head" + assert call_kwargs["database_url"] == "postgres://test" + + +def test_run_migrate_failure_raises_exit(mock_runtime, mock_connection, mock_settings): + """Test that migration failure raises typer.Exit.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + with patch("src.cli.commands.db.workflows.run_migration") as mock_migrate: + mock_migrate.return_value = False + + with pytest.raises(typer.Exit) as exc_info: + run_migrate( + mock_runtime, + action="upgrade", + revision=None, + message="test migration", + merge_revisions=[], + purge=False, + autogenerate=True, + sql=False, + ) + + assert exc_info.value.exit_code == 1 + + +def test_workflows_use_port_forward_context( + mock_runtime, mock_connection, mock_settings +): + """Test that all workflows properly use port_forward context manager.""" + mock_runtime.get_settings.return_value = mock_settings + mock_runtime.connect.return_value = mock_connection + + # Track if port_forward context was entered + port_forward_entered = False + + def track_port_forward(): + nonlocal port_forward_entered + port_forward_entered = True + return no_port_forward() + + mock_runtime.port_forward = track_port_forward + + with patch("src.infra.postgres.PostgresInitializer") as mock_init: + mock_init.return_value.initialize.return_value = True + run_init(mock_runtime) + assert port_forward_entered + + port_forward_entered = False + with patch("src.infra.postgres.PostgresVerifier") as mock_verify: + mock_verify.return_value.verify_all.return_value = True + run_verify(mock_runtime, superuser_mode=True) + assert port_forward_entered diff --git a/uv.lock b/uv.lock index 0d2d1de..7e3d764 100644 --- a/uv.lock +++ b/uv.lock @@ -15,6 +15,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/a9/0da089c3ae7a31cbcd2dcf0214f6f571e1295d292b6139e2bac68ec081d0/aioredis-2.0.1-py3-none-any.whl", hash = "sha256:9ac0d0b3b485d293b8ca1987e6de8658d7dafcca1cddfcd1d506cae8cdebfdd6", size = 71243, upload-time = "2021-12-27T20:28:16.36Z" }, ] +[[package]] +name = "alembic" +version = "1.17.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/a6/74c8cadc2882977d80ad756a13857857dbcf9bd405bc80b662eb10651282/alembic-1.17.2.tar.gz", hash = "sha256:bbe9751705c5e0f14877f02d46c53d10885e377e3d90eda810a016f9baa19e8e", size = 1988064, upload-time = "2025-11-14T20:35:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl", hash = "sha256:f483dd1fe93f6c5d49217055e4d15b905b425b6af906746abb35b69c1996c4e6", size = 248554, upload-time = "2025-11-14T20:35:05.699Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -43,8 +57,10 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "aioredis" }, + { name = "alembic" }, { name = "authlib" }, { name = "cachetools" }, + { name = "click" }, { name = "fastapi" }, { name = "fastapi-limiter" }, { name = "httpx" }, @@ -77,13 +93,16 @@ dev = [ { name = "pytest-xdist" }, { name = "ruff" }, { name = "types-authlib" }, + { name = "types-psycopg2" }, ] [package.metadata] requires-dist = [ { name = "aioredis", specifier = ">=2.0.0" }, + { name = "alembic", specifier = ">=1.13.0" }, { name = "authlib", specifier = ">=1.6.4" }, { name = "cachetools", specifier = ">=5.3.3" }, + { name = "click", specifier = ">=8.2.1" }, { name = "fastapi", specifier = ">=0.116.1" }, { name = "fastapi-limiter", specifier = ">=0.1.6" }, { name = "httpx", specifier = ">=0.27.2" }, @@ -116,6 +135,7 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.5.0" }, { name = "ruff", specifier = ">=0.1.0" }, { name = "types-authlib", specifier = ">=1.6.2.20250914" }, + { name = "types-psycopg2", specifier = ">=2.9.21.20251012" }, ] [[package]] @@ -691,6 +711,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, ] +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1378,6 +1410,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/57/3a0d89b33b7485b7ffd99ec7cf53b0c5c89194c481f0bd673fd67e5f273f/types_protobuf-6.32.1.20251105-py3-none-any.whl", hash = "sha256:a15109d38f7cfefd2539ef86d3f93a6a41c7cad53924f8aa1a51eaddbb72a660", size = 77890, upload-time = "2025-11-05T03:04:42.067Z" }, ] +[[package]] +name = "types-psycopg2" +version = "2.9.21.20251012" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/b3/2d09eaf35a084cffd329c584970a3fa07101ca465c13cad1576d7c392587/types_psycopg2-2.9.21.20251012.tar.gz", hash = "sha256:4cdafd38927da0cfde49804f39ab85afd9c6e9c492800e42f1f0c1a1b0312935", size = 26710, upload-time = "2025-10-12T02:55:39.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/0c/05feaf8cb51159f2c0af04b871dab7e98a2f83a3622f5f216331d2dd924c/types_psycopg2-2.9.21.20251012-py3-none-any.whl", hash = "sha256:712bad5c423fe979e357edbf40a07ca40ef775d74043de72bd4544ca328cc57e", size = 24883, upload-time = "2025-10-12T02:55:38.439Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 12d3624fd57601457252183bdaeb91994bd5439b Mon Sep 17 00:00:00 2001 From: jc Date: Thu, 1 Jan 2026 20:37:27 -0500 Subject: [PATCH 2/5] Fixing CI tests --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d47b59..8d828fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: - name: Start dev environment services run: | # Deploy dev environment (blocks until services are ready) - uv run api-forge-cli deploy up --no-start-server dev + uv run api-forge-cli dev up --no-start-server - name: Run tests run: | @@ -70,7 +70,7 @@ jobs: - name: Stop dev environment if: always() run: | - uv run api-forge-cli deploy down dev + uv run api-forge-cli dev down - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 3bd91a92eaaa7c0fa2f05b9d7f0db4c4e8f61d7f Mon Sep 17 00:00:00 2001 From: jc Date: Thu, 1 Jan 2026 20:45:57 -0500 Subject: [PATCH 3/5] Fixing tests --- tests/unit/cli/deployment/test_image_builder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/cli/deployment/test_image_builder.py b/tests/unit/cli/deployment/test_image_builder.py index 04c09da..1885a4c 100644 --- a/tests/unit/cli/deployment/test_image_builder.py +++ b/tests/unit/cli/deployment/test_image_builder.py @@ -254,14 +254,19 @@ def test_full_build_flow_for_minikube(self, full_project_setup: Path) -> None: mock_commands.docker.compose_build.return_value = None mock_commands.docker.tag_image.return_value = MagicMock(success=True) mock_commands.docker.image_exists.return_value = False - mock_commands.kubectl.is_minikube_context.return_value = True mock_commands.docker.minikube_load_image.return_value = None + # Mock the controller to return minikube context + mock_controller = MagicMock() + mock_controller.is_minikube_context.return_value = True + mock_controller.get_current_context.return_value = "minikube" + mock_console = MagicMock() builder = ImageBuilder( commands=mock_commands, console=mock_console, + controller=mock_controller, paths=DeploymentPaths(project_root=full_project_setup), ) From d3d4592cd8e3ffc4447153b6aea5d1f6f59dde14 Mon Sep 17 00:00:00 2001 From: jc Date: Thu, 1 Jan 2026 20:52:18 -0500 Subject: [PATCH 4/5] Fixing tests --- tests/integration/test_application_startup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/integration/test_application_startup.py b/tests/integration/test_application_startup.py index eb79d05..4c09e18 100644 --- a/tests/integration/test_application_startup.py +++ b/tests/integration/test_application_startup.py @@ -45,6 +45,10 @@ class DummyLimiter: async def init(redis): init_called["called"] = True + # Mock init_db to avoid database conflicts in parallel tests + def mock_init_db(): + pass + # Monkeypatch dependencies monkeypatch.setattr(application, "FastAPILimiter", DummyLimiter) monkeypatch.setattr( @@ -52,6 +56,7 @@ async def init(redis): "redis_async", type("_m", (), {"from_url": staticmethod(fake_from_url)}), ) + monkeypatch.setattr(application, "init_db", mock_init_db) # Call startup within the test context with with_context(config_override=test_config): @@ -76,9 +81,14 @@ async def test_startup_initializes_rate_limiter_without_redis( test_config.redis.url = "redis://localhost:6379/0" test_config.redis.enabled = False # Disable Redis + # Mock init_db to avoid database conflicts in parallel tests + def mock_init_db(): + pass + # Simulate missing dependencies monkeypatch.setattr(application, "FastAPILimiter", None) monkeypatch.setattr(application, "redis_async", None) + monkeypatch.setattr(application, "init_db", mock_init_db) with with_context(config_override=test_config): await application.startup() From b730f0b57d42383f7e1d783e33977f96ddfde215 Mon Sep 17 00:00:00 2001 From: jc Date: Thu, 1 Jan 2026 20:56:41 -0500 Subject: [PATCH 5/5] Fixing tests --- tests/integration/test_application_startup.py | 4 ++-- tests/integration/test_oidc_compliance.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_application_startup.py b/tests/integration/test_application_startup.py index 4c09e18..ee241f7 100644 --- a/tests/integration/test_application_startup.py +++ b/tests/integration/test_application_startup.py @@ -56,7 +56,7 @@ def mock_init_db(): "redis_async", type("_m", (), {"from_url": staticmethod(fake_from_url)}), ) - monkeypatch.setattr(application, "init_db", mock_init_db) + monkeypatch.setattr("src.app.runtime.init_db.init_db", mock_init_db) # Call startup within the test context with with_context(config_override=test_config): @@ -88,7 +88,7 @@ def mock_init_db(): # Simulate missing dependencies monkeypatch.setattr(application, "FastAPILimiter", None) monkeypatch.setattr(application, "redis_async", None) - monkeypatch.setattr(application, "init_db", mock_init_db) + monkeypatch.setattr("src.app.runtime.init_db.init_db", mock_init_db) with with_context(config_override=test_config): await application.startup() diff --git a/tests/integration/test_oidc_compliance.py b/tests/integration/test_oidc_compliance.py index f7605cd..b8371f5 100644 --- a/tests/integration/test_oidc_compliance.py +++ b/tests/integration/test_oidc_compliance.py @@ -11,8 +11,15 @@ @pytest.fixture -def client(): +def client(monkeypatch): """Create a test client for the FastAPI app.""" + + # Mock init_db to avoid database conflicts in parallel tests + def mock_init_db(): + pass + + monkeypatch.setattr("src.app.runtime.init_db.init_db", mock_init_db) + with TestClient(app) as test_client: yield test_client