diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0a544190dd..2ac477e00e 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -88,3 +88,10 @@ jobs: target_environment: staging vercel_project_id_var: VERCEL_PROJECT_ID_GLOBAL_APP secrets: inherit + + deploy-workers: + needs: [check-staging-db-startup, run-migrations, deploy-app, deploy-global-app] + uses: ./.github/workflows/deploy-workers.yml + with: + target_environment: staging + secrets: inherit diff --git a/.github/workflows/deploy-workers.yml b/.github/workflows/deploy-workers.yml index 1e18146a03..45d99badbd 100644 --- a/.github/workflows/deploy-workers.yml +++ b/.github/workflows/deploy-workers.yml @@ -7,11 +7,30 @@ on: description: 'Worker folder to deploy (e.g. services/app-builder)' required: true type: string + target_environment: + description: 'Worker environment to deploy to' + required: false + default: production + type: choice + options: + - production + - staging workflow_call: inputs: + worker: + description: 'Worker folder to deploy for single-worker calls' + required: false + default: '' + type: string base_sha: description: 'Base SHA to diff against when detecting changed workers' - required: true + required: false + default: '' + type: string + target_environment: + description: 'Worker environment to deploy to' + required: false + default: production type: string permissions: @@ -23,10 +42,11 @@ concurrency: jobs: deploy-manual: - if: github.event_name == 'workflow_dispatch' + if: inputs.worker != '' runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} timeout-minutes: 15 name: Deploy ${{ inputs.worker }} + environment: ${{ inputs.target_environment }} steps: - name: Checkout code uses: useblacksmith/checkout@41cdeedae8edb2e684ba22896a5fd2a3cb85db6b # v1 @@ -53,10 +73,10 @@ jobs: # right before deploy; all other workers are unaffected. preCommands: | if [ "$(jq -r '.scripts.predeploy // empty' package.json)" != "" ]; then pnpm run predeploy; fi - command: deploy + command: ${{ inputs.target_environment == 'production' && 'deploy' || format('deploy --env {0}', inputs.target_environment) }} detect-changes: - if: inputs.base_sha != '' + if: inputs.worker == '' && (inputs.base_sha != '' || inputs.target_environment != 'production') runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} timeout-minutes: 5 outputs: @@ -73,10 +93,68 @@ jobs: # Auto-discover all deployable workers (folders containing wrangler.jsonc) # and exclude workers that have their own deploy pipelines. # - # Diff against the SHA before the push so that multi-commit pushes - # (e.g. a merge commit that squashes several commits) don't miss workers - # that were only touched in earlier commits of the same push. + # Production diffs against the SHA before the push so that multi-commit + # pushes don't miss workers touched in earlier commits. Staging deploys + # only workers that explicitly define the named Wrangler environment. BASE_SHA="${{ inputs.base_sha }}" + TARGET_ENVIRONMENT="${{ inputs.target_environment }}" + + has_named_environment() { + node - "$1" "$TARGET_ENVIRONMENT" <<'NODE' + const fs = require('node:fs'); + + const [file, targetEnvironment] = process.argv.slice(2); + + function stripJsonComments(value) { + let output = ''; + let inString = false; + let escaped = false; + + for (let index = 0; index < value.length; index += 1) { + const current = value[index]; + const next = value[index + 1]; + + if (inString) { + output += current; + if (escaped) escaped = false; + else if (current === '\\') escaped = true; + else if (current === '"') inString = false; + continue; + } + + if (current === '"') { + inString = true; + output += current; + continue; + } + + if (current === '/' && next === '/') { + while (index < value.length && value[index] !== '\n') index += 1; + output += '\n'; + continue; + } + + if (current === '/' && next === '*') { + index += 2; + while (index < value.length && !(value[index] === '*' && value[index + 1] === '/')) { + if (value[index] === '\n') output += '\n'; + index += 1; + } + index += 1; + continue; + } + + output += current; + } + + return output; + } + + const rawConfig = fs.readFileSync(file, 'utf8'); + const config = JSON.parse(stripJsonComments(rawConfig).replace(/,\s*([}\]])/g, '$1')); + process.exit(config.env && Object.hasOwn(config.env, targetEnvironment) ? 0 : 1); + NODE + } # Workers excluded from this workflow (they have custom deploy pipelines): EXCLUDED=( @@ -108,12 +186,23 @@ jobs: fi done - CHANGED=() - for dir in "${DEPLOYABLE[@]}"; do - if git diff --name-only "$BASE_SHA" HEAD -- "$dir/" | grep -q .; then - CHANGED+=("$dir") - fi - done + if [ "$TARGET_ENVIRONMENT" = "production" ]; then + CHANGED=() + for dir in "${DEPLOYABLE[@]}"; do + if git diff --name-only "$BASE_SHA" HEAD -- "$dir/" | grep -q .; then + CHANGED+=("$dir") + fi + done + else + CHANGED=() + for dir in "${DEPLOYABLE[@]}"; do + if has_named_environment "$dir/wrangler.jsonc"; then + CHANGED+=("$dir") + else + echo "Skipping $dir: wrangler.jsonc does not define env.$TARGET_ENVIRONMENT" + fi + done + fi if [ ${#CHANGED[@]} -eq 0 ]; then echo "matrix=[]" >> "$GITHUB_OUTPUT" @@ -127,6 +216,7 @@ jobs: if: needs.detect-changes.outputs.matrix != '[]' && needs.detect-changes.outputs.matrix != '' runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} timeout-minutes: 15 + environment: ${{ inputs.target_environment }} strategy: fail-fast: false matrix: @@ -158,4 +248,4 @@ jobs: # right before deploy; all other workers are unaffected. preCommands: | if [ "$(jq -r '.scripts.predeploy // empty' package.json)" != "" ]; then pnpm run predeploy; fi - command: deploy + command: ${{ inputs.target_environment == 'production' && 'deploy' || format('deploy --env {0}', inputs.target_environment) }} diff --git a/services/cloud-agent-next/wrangler.jsonc b/services/cloud-agent-next/wrangler.jsonc index 5518b09758..ebadf70ed8 100644 --- a/services/cloud-agent-next/wrangler.jsonc +++ b/services/cloud-agent-next/wrangler.jsonc @@ -265,6 +265,148 @@ * https://developers.cloudflare.com/workers/wrangler/configuration/#named-environments */ "env": { + "staging": { + "name": "cloud-agent-next-staging", + "workers_dev": true, + "preview_urls": false, + "triggers": { + "crons": ["17 2 * * *"], + }, + "routes": [], + "hyperdrive": [ + { + "binding": "HYPERDRIVE", + "id": "624ec80650dd414199349f4e217ddb10", + "localConnectionString": "postgres://postgres:postgres@localhost:5432/postgres", + }, + ], + "vars": { + "KILOCODE_BACKEND_BASE_URL": "https://staging-app.kilo.ai", + "KILO_OPENROUTER_BASE": "https://staging-app.kilo.ai/api", + "GITHUB_APP_SLUG": "kiloconnect-development", + "GITHUB_APP_BOT_USER_ID": "242397087", + "GITHUB_LITE_APP_SLUG": "", + "GITHUB_LITE_APP_BOT_USER_ID": "", + "WORKER_URL": "https://cloud-agent-next-staging.engineering-e11.workers.dev", + "CLI_TIMEOUT_SECONDS": "900", + "REAPER_INTERVAL_MS": "300000", + "R2_ATTACHMENTS_BUCKET": "cloud-agent-attachments-staging", + "WS_ALLOWED_ORIGINS": "https://staging-app.kilo.ai", + "KILO_SESSION_INGEST_URL": "https://session-ingest-staging.engineering-e11.workers.dev", + "PER_SESSION_SANDBOX_ORG_IDS": "*", + }, + "services": [ + { + "binding": "SESSION_INGEST", + "service": "session-ingest-staging", + "entrypoint": "SessionIngestRPC", + }, + { + "binding": "GIT_TOKEN_SERVICE", + "service": "git-token-service-staging", + "entrypoint": "GitTokenRPCEntrypoint", + }, + { + "binding": "NOTIFICATIONS", + "service": "notifications-staging", + "entrypoint": "NotificationsService", + }, + ], + "secrets_store_secrets": [ + { + "binding": "INTERNAL_API_SECRET_PROD", + "store_id": "342a86d9e3a94da698e82d0c6e2a36f0", + "secret_name": "INTERNAL_API_SECRET_PROD", + }, + ], + "r2_buckets": [ + { + "binding": "R2_BUCKET", + "bucket_name": "kilocode-sessions-staging", + }, + ], + "containers": [ + { + "class_name": "Sandbox", + "image": "./Dockerfile", + "instance_type": "standard-4", + "image_vars": { + "KILOCODE_CLI_VERSION": "7.3.21", + }, + "max_instances": 10, + "rollout_active_grace_period": 300, + }, + { + "class_name": "SandboxSmall", + "image": "./Dockerfile", + "instance_type": "standard-4", + "image_vars": { + "KILOCODE_CLI_VERSION": "7.3.21", + }, + "max_instances": 2, + "rollout_active_grace_period": 300, + }, + { + "class_name": "SandboxDIND", + "image": "./Dockerfile.dind", + "instance_type": "standard-3", + "image_vars": { + "KILOCODE_CLI_VERSION": "7.3.21", + }, + "max_instances": 1, + "rollout_active_grace_period": 300, + }, + ], + "durable_objects": { + "bindings": [ + { + "class_name": "Sandbox", + "name": "Sandbox", + }, + { + "class_name": "SandboxSmall", + "name": "SandboxSmall", + }, + { + "class_name": "SandboxDIND", + "name": "SandboxDIND", + }, + { + "class_name": "CloudAgentSession", + "name": "CLOUD_AGENT_SESSION", + }, + { + "class_name": "UserKiloFacade", + "name": "USER_KILO_FACADE", + }, + ], + }, + "queues": { + "producers": [ + { + "binding": "CALLBACK_QUEUE", + "queue": "cloud-agent-next-callback-queue-staging", + }, + { + "binding": "CLOUD_AGENT_REPORT_QUEUE", + "queue": "cloud-agent-next-report-queue-staging", + }, + ], + "consumers": [ + { + "queue": "cloud-agent-next-callback-queue-staging", + "max_batch_size": 5, + // Keep aligned with CALLBACK_DELIVERY_MAX_ATTEMPTS = initial attempt + 4 redeliveries. + "max_retries": 4, + }, + { + "queue": "cloud-agent-next-report-queue-staging", + "max_retries": 3, + "dead_letter_queue": "cloud-agent-next-report-queue-dlq-staging", + }, + ], + }, + }, "dev": { "name": "cloud-agent-next-dev", "triggers": {