Add TinyBird credentials section to token rotation runbook (#154) #73
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GHO-26: Health checks after deployment | |
| name: Deploy to Dev Environment | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| packages: read | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| environment: dev | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Required if the GHCR image is private | |
| - name: Log in to GHCR | |
| env: | |
| GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} | |
| run: | | |
| docker login ghcr.io -u "noahwhite" --password-stdin <<< "$GHCR_TOKEN" | |
| - name: Pull deployment tools image | |
| run: | | |
| docker pull ghcr.io/noahwhite/ghost-stack-shell:latest | |
| - name: Verify deployment context | |
| run: | | |
| echo "✅ Deployment workflow triggered for develop branch" | |
| echo " Environment: dev" | |
| echo " Working directory: ${GITHUB_WORKSPACE}" | |
| echo " Runner: $(uname -a)" | |
| - name: Verify tooling availability | |
| run: | | |
| docker run --rm \ | |
| -v "${GITHUB_WORKSPACE}:/home/devops/app" \ | |
| -w /home/devops/app \ | |
| ghcr.io/noahwhite/ghost-stack-shell:latest \ | |
| bash -c "git config --global --add safe.directory /home/devops/app && echo '✅ OpenTofu tooling verified' && tofu version" | |
| - name: Extract PR number from merge commit | |
| id: pr | |
| env: | |
| COMMIT_MSG: ${{ github.event.head_commit.message }} | |
| run: | | |
| PR_NUM=$(echo "$COMMIT_MSG" | grep -oP '\(#\K\d+(?=\))' | head -1) | |
| if [ -z "$PR_NUM" ]; then | |
| echo "❌ No PR number found in commit message" | |
| echo " This deployment requires a PR merge commit" | |
| exit 1 | |
| fi | |
| echo "number=$PR_NUM" >> $GITHUB_OUTPUT | |
| echo "✅ Found PR #$PR_NUM" | |
| - name: Download plan artifact from PR | |
| id: download-artifact | |
| uses: dawidd6/action-download-artifact@v9 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| workflow: pr-tofu-plan-develop.yml | |
| pr: ${{ steps.pr.outputs.number }} | |
| name: tofu-plan-dev-${{ steps.pr.outputs.number }} | |
| path: ./pr-plan | |
| if_no_artifact_found: warn | |
| - name: Check if deployment should proceed | |
| id: skip-check | |
| run: | | |
| # Check if plan artifact exists (no artifact = no infra changes in PR) | |
| if [ ! -f ./pr-plan/plan-output.txt ]; then | |
| echo "✅ No plan artifact found for PR #${{ steps.pr.outputs.number }}" | |
| echo " No infrastructure files were changed in this PR." | |
| echo " Skipping deployment." | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "✅ Plan artifact verified" | |
| echo "Plan from PR #${{ steps.pr.outputs.number }}:" | |
| head -20 ./pr-plan/plan-output.txt | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install Bitwarden Secrets Manager CLI (bws) | |
| if: steps.skip-check.outputs.skip != 'true' | |
| run: | | |
| set -euo pipefail | |
| curl -fsSL \ | |
| https://github.com/bitwarden/sdk-sm/releases/download/bws-v1.0.0/bws-x86_64-unknown-linux-gnu-1.0.0.zip \ | |
| -o /tmp/bws.zip | |
| unzip -q /tmp/bws.zip -d /tmp/bws | |
| sudo mv /tmp/bws/bws /usr/local/bin/bws | |
| sudo chmod +x /usr/local/bin/bws | |
| bws --version | |
| - name: Retrieve secrets via infra-shell.sh (CI mode) | |
| if: steps.skip-check.outputs.skip != 'true' | |
| env: | |
| BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }} | |
| BOOTSTRAP_R2_BUCKET: ${{ vars.BOOTSTRAP_R2_BUCKET }} | |
| ADMIN_IP: ${{ secrets.ADMIN_IP }} | |
| CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} | |
| run: | | |
| ./docker/scripts/infra-shell.sh --ci --secrets-only --export-github-env | |
| - name: Fix workspace permissions for Docker container | |
| if: steps.skip-check.outputs.skip != 'true' | |
| run: | | |
| chmod -R a+w "${GITHUB_WORKSPACE}/opentofu" | |
| - name: Run OpenTofu init | |
| if: steps.skip-check.outputs.skip != 'true' | |
| env: | |
| TF_VAR_cloudflare_account_id: ${{ env.TF_VAR_cloudflare_account_id }} | |
| TF_VAR_cloudflare_api_token: ${{ env.TF_VAR_cloudflare_api_token }} | |
| R2_ACCESS_KEY_ID: ${{ env.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ env.R2_SECRET_ACCESS_KEY }} | |
| TF_VAR_vultr_api_key: ${{ env.TF_VAR_vultr_api_key }} | |
| TAILSCALE_API_KEY: ${{ env.TAILSCALE_API_KEY }} | |
| TAILSCALE_TAILNET: ${{ env.TAILSCALE_TAILNET }} | |
| TF_VAR_PD_CLIENT_ID: ${{ env.TF_VAR_PD_CLIENT_ID }} | |
| TF_VAR_PD_CLIENT_SECRET: ${{ env.TF_VAR_PD_CLIENT_SECRET }} | |
| TF_VAR_pd_subdomain: ${{ env.TF_VAR_pd_subdomain }} | |
| TF_VAR_pd_user_tok: ${{ env.TF_VAR_pd_user_tok }} | |
| TF_VAR_GC_ACCESS_TOK: ${{ env.TF_VAR_GC_ACCESS_TOK }} | |
| TF_VAR_SOC_DEV_TERRAFORM_SA_TOK: ${{ env.TF_VAR_SOC_DEV_TERRAFORM_SA_TOK }} | |
| TF_BACKEND_BUCKET: ${{ env.TF_BACKEND_BUCKET }} | |
| run: | | |
| docker run --rm \ | |
| -v "${GITHUB_WORKSPACE}:/home/devops/app" \ | |
| -w /home/devops/app \ | |
| -e TF_VAR_cloudflare_account_id \ | |
| -e TF_VAR_cloudflare_api_token \ | |
| -e R2_ACCESS_KEY_ID \ | |
| -e R2_SECRET_ACCESS_KEY \ | |
| -e TF_VAR_vultr_api_key \ | |
| -e TAILSCALE_API_KEY \ | |
| -e TAILSCALE_TAILNET \ | |
| -e TF_VAR_PD_CLIENT_ID \ | |
| -e TF_VAR_PD_CLIENT_SECRET \ | |
| -e TF_VAR_pd_subdomain \ | |
| -e TF_VAR_pd_user_tok \ | |
| -e TF_VAR_GC_ACCESS_TOK \ | |
| -e TF_VAR_SOC_DEV_TERRAFORM_SA_TOK \ | |
| -e TF_BACKEND_BUCKET \ | |
| ghcr.io/noahwhite/ghost-stack-shell:latest \ | |
| bash -c "git config --global --add safe.directory /home/devops/app && ./opentofu/scripts/tofu.sh dev init" | |
| - name: Generate fresh plan for comparison | |
| if: steps.skip-check.outputs.skip != 'true' | |
| env: | |
| TF_VAR_cloudflare_account_id: ${{ env.TF_VAR_cloudflare_account_id }} | |
| TF_VAR_cloudflare_api_token: ${{ env.TF_VAR_cloudflare_api_token }} | |
| R2_ACCESS_KEY_ID: ${{ env.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ env.R2_SECRET_ACCESS_KEY }} | |
| TF_VAR_vultr_api_key: ${{ env.TF_VAR_vultr_api_key }} | |
| TAILSCALE_API_KEY: ${{ env.TAILSCALE_API_KEY }} | |
| TAILSCALE_TAILNET: ${{ env.TAILSCALE_TAILNET }} | |
| TF_VAR_PD_CLIENT_ID: ${{ env.TF_VAR_PD_CLIENT_ID }} | |
| TF_VAR_PD_CLIENT_SECRET: ${{ env.TF_VAR_PD_CLIENT_SECRET }} | |
| TF_VAR_pd_subdomain: ${{ env.TF_VAR_pd_subdomain }} | |
| TF_VAR_pd_user_tok: ${{ env.TF_VAR_pd_user_tok }} | |
| TF_VAR_GC_ACCESS_TOK: ${{ env.TF_VAR_GC_ACCESS_TOK }} | |
| TF_VAR_SOC_DEV_TERRAFORM_SA_TOK: ${{ env.TF_VAR_SOC_DEV_TERRAFORM_SA_TOK }} | |
| TF_BACKEND_BUCKET: ${{ env.TF_BACKEND_BUCKET }} | |
| TF_VAR_admin_subnets: ${{ env.TF_VAR_admin_subnets }} | |
| TF_VAR_admin_ip: ${{ env.TF_VAR_admin_ip }} | |
| TF_VAR_ssh_public_key: ${{ env.TF_VAR_ssh_public_key }} | |
| TF_VAR_cloudflare_zone_id: ${{ env.TF_VAR_cloudflare_zone_id }} | |
| run: | | |
| docker run --rm \ | |
| -v "${GITHUB_WORKSPACE}:/home/devops/app" \ | |
| -w /home/devops/app \ | |
| -e TF_VAR_cloudflare_account_id \ | |
| -e TF_VAR_cloudflare_api_token \ | |
| -e R2_ACCESS_KEY_ID \ | |
| -e R2_SECRET_ACCESS_KEY \ | |
| -e TF_VAR_vultr_api_key \ | |
| -e TAILSCALE_API_KEY \ | |
| -e TAILSCALE_TAILNET \ | |
| -e TF_VAR_PD_CLIENT_ID \ | |
| -e TF_VAR_PD_CLIENT_SECRET \ | |
| -e TF_VAR_pd_subdomain \ | |
| -e TF_VAR_pd_user_tok \ | |
| -e TF_VAR_GC_ACCESS_TOK \ | |
| -e TF_VAR_SOC_DEV_TERRAFORM_SA_TOK \ | |
| -e TF_BACKEND_BUCKET \ | |
| -e TF_VAR_admin_subnets \ | |
| -e TF_VAR_admin_ip \ | |
| -e TF_VAR_ssh_public_key \ | |
| -e TF_VAR_cloudflare_zone_id \ | |
| ghcr.io/noahwhite/ghost-stack-shell:latest \ | |
| bash -c "git config --global --add safe.directory /home/devops/app && ./opentofu/scripts/tofu.sh dev plan -out=tfplan" | |
| - name: Generate human-readable current plan | |
| if: steps.skip-check.outputs.skip != 'true' | |
| run: | | |
| docker run --rm \ | |
| -v "${GITHUB_WORKSPACE}:/home/devops/app" \ | |
| -w /home/devops/app \ | |
| ghcr.io/noahwhite/ghost-stack-shell:latest \ | |
| bash -c "git config --global --add safe.directory /home/devops/app && tofu -chdir=opentofu/envs/dev show -no-color tfplan > opentofu/envs/dev/current-plan.txt" | |
| - name: Compare PR plan with current plan | |
| if: steps.skip-check.outputs.skip != 'true' | |
| run: | | |
| echo "Comparing PR plan with current state..." | |
| # Filter out ephemeral values that change between plan runs | |
| # - kvm: Vultr KVM console URL contains session tokens that regenerate on every API call | |
| # - allowed_bandwidth: Vultr bandwidth allocation changes over time (monthly reset/accumulation) | |
| filter_ephemeral() { | |
| grep -v '~ kvm' | grep -v '~ allowed_bandwidth' | |
| } | |
| filter_ephemeral < ./pr-plan/plan-output.txt > /tmp/pr-plan-filtered.txt | |
| filter_ephemeral < ./opentofu/envs/dev/current-plan.txt > /tmp/current-plan-filtered.txt | |
| if diff -u /tmp/pr-plan-filtered.txt /tmp/current-plan-filtered.txt; then | |
| echo "✅ Plans match - no drift detected" | |
| echo "PLANS_MATCH=true" >> $GITHUB_ENV | |
| else | |
| echo "❌ Plans differ - state has drifted since PR approval" | |
| echo " PR was approved based on a different plan" | |
| echo " Manual review required" | |
| echo "PLANS_MATCH=false" >> $GITHUB_ENV | |
| exit 1 | |
| fi | |
| - name: Apply infrastructure changes | |
| if: steps.skip-check.outputs.skip != 'true' && env.PLANS_MATCH == 'true' | |
| env: | |
| TF_VAR_cloudflare_account_id: ${{ env.TF_VAR_cloudflare_account_id }} | |
| TF_VAR_cloudflare_api_token: ${{ env.TF_VAR_cloudflare_api_token }} | |
| R2_ACCESS_KEY_ID: ${{ env.R2_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ env.R2_SECRET_ACCESS_KEY }} | |
| TF_VAR_vultr_api_key: ${{ env.TF_VAR_vultr_api_key }} | |
| TAILSCALE_API_KEY: ${{ env.TAILSCALE_API_KEY }} | |
| TAILSCALE_TAILNET: ${{ env.TAILSCALE_TAILNET }} | |
| TF_VAR_PD_CLIENT_ID: ${{ env.TF_VAR_PD_CLIENT_ID }} | |
| TF_VAR_PD_CLIENT_SECRET: ${{ env.TF_VAR_PD_CLIENT_SECRET }} | |
| TF_VAR_pd_subdomain: ${{ env.TF_VAR_pd_subdomain }} | |
| TF_VAR_pd_user_tok: ${{ env.TF_VAR_pd_user_tok }} | |
| TF_VAR_GC_ACCESS_TOK: ${{ env.TF_VAR_GC_ACCESS_TOK }} | |
| TF_VAR_SOC_DEV_TERRAFORM_SA_TOK: ${{ env.TF_VAR_SOC_DEV_TERRAFORM_SA_TOK }} | |
| TF_BACKEND_BUCKET: ${{ env.TF_BACKEND_BUCKET }} | |
| TF_VAR_admin_subnets: ${{ env.TF_VAR_admin_subnets }} | |
| TF_VAR_admin_ip: ${{ env.TF_VAR_admin_ip }} | |
| TF_VAR_ssh_public_key: ${{ env.TF_VAR_ssh_public_key }} | |
| TF_VAR_cloudflare_zone_id: ${{ env.TF_VAR_cloudflare_zone_id }} | |
| run: | | |
| docker run --rm \ | |
| -v "${GITHUB_WORKSPACE}:/home/devops/app" \ | |
| -w /home/devops/app \ | |
| -e TF_VAR_cloudflare_account_id \ | |
| -e TF_VAR_cloudflare_api_token \ | |
| -e R2_ACCESS_KEY_ID \ | |
| -e R2_SECRET_ACCESS_KEY \ | |
| -e TF_VAR_vultr_api_key \ | |
| -e TAILSCALE_API_KEY \ | |
| -e TAILSCALE_TAILNET \ | |
| -e TF_VAR_PD_CLIENT_ID \ | |
| -e TF_VAR_PD_CLIENT_SECRET \ | |
| -e TF_VAR_pd_subdomain \ | |
| -e TF_VAR_pd_user_tok \ | |
| -e TF_VAR_GC_ACCESS_TOK \ | |
| -e TF_VAR_SOC_DEV_TERRAFORM_SA_TOK \ | |
| -e TF_BACKEND_BUCKET \ | |
| -e TF_VAR_admin_subnets \ | |
| -e TF_VAR_admin_ip \ | |
| -e TF_VAR_ssh_public_key \ | |
| -e TF_VAR_cloudflare_zone_id \ | |
| ghcr.io/noahwhite/ghost-stack-shell:latest \ | |
| bash -c "git config --global --add safe.directory /home/devops/app && ./opentofu/scripts/tofu.sh dev apply tfplan" | |
| - name: Verify Ghost health after deployment | |
| if: steps.skip-check.outputs.skip != 'true' && env.PLANS_MATCH == 'true' | |
| env: | |
| HEALTH_CHECK_TOKEN: ${{ secrets.HEALTH_CHECK_TOKEN }} | |
| run: | | |
| echo "🏥 Starting health checks for Ghost deployment..." | |
| MAX_RETRIES=30 | |
| RETRY_INTERVAL=10 | |
| URL="https://separationofconcerns.dev" | |
| for i in $(seq 1 $MAX_RETRIES); do | |
| echo "Health check attempt $i of $MAX_RETRIES..." | |
| HTTP_CODE=$(curl -sSL --max-time 10 -o /dev/null -w "%{http_code}" \ | |
| -H "X-Health-Check-Token: ${HEALTH_CHECK_TOKEN}" \ | |
| "$URL" 2>&1 || echo "000") | |
| if [ "$HTTP_CODE" = "200" ]; then | |
| echo "✅ Ghost is healthy - received HTTP $HTTP_CODE" | |
| echo "HEALTH_CHECK_PASSED=true" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| echo " Received HTTP $HTTP_CODE, waiting ${RETRY_INTERVAL}s before retry..." | |
| sleep $RETRY_INTERVAL | |
| done | |
| echo "❌ Health check failed after $MAX_RETRIES attempts ($(($MAX_RETRIES * $RETRY_INTERVAL))s timeout)" | |
| echo " Last HTTP response code: $HTTP_CODE" | |
| exit 1 | |
| - name: Deployment summary | |
| if: always() | |
| run: | | |
| echo "================================" | |
| echo "Deployment Summary" | |
| echo "================================" | |
| if [ "${{ steps.skip-check.outputs.skip }}" == "true" ]; then | |
| echo "⏭️ Deployment skipped - no infrastructure changes in PR" | |
| echo " PR #${{ steps.pr.outputs.number }} merged without infrastructure deployment" | |
| elif [ "${{ env.PLANS_MATCH }}" == "true" ] && [ "${{ env.HEALTH_CHECK_PASSED }}" == "true" ]; then | |
| echo "✅ Infrastructure changes applied successfully" | |
| echo "✅ Ghost health check passed" | |
| echo " PR #${{ steps.pr.outputs.number }} deployed to dev environment" | |
| echo " URL: https://separationofconcerns.dev" | |
| elif [ "${{ env.PLANS_MATCH }}" == "true" ]; then | |
| echo "✅ Infrastructure changes applied" | |
| echo "❌ Ghost health check failed" | |
| echo " PR #${{ steps.pr.outputs.number }} - deployment may need investigation" | |
| else | |
| echo "❌ Deployment failed - plans did not match" | |
| echo " State drift detected since PR approval" | |
| fi |