From 6250c862b2ec6ec109b70374c772fd5d9708ac52 Mon Sep 17 00:00:00 2001 From: Riley Seaburg Date: Tue, 13 Jan 2026 17:56:55 +0000 Subject: [PATCH 1/2] Scale down web app instances during rolling deploy to avoid memory quota exceeded Similar to the sidekiq worker fix in commit 48b4e42d, scale down the main touchpoints web app to 1 instance before deploying, then scale back up after successful deployment. This prevents 'organization's memory limit exceeded' errors during rolling deployments when the org doesn't have enough quota for multiple instances during the staging/deployment process. --- .circleci/deploy.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index a70ce758c..82483c687 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -221,6 +221,17 @@ cf_push_with_retry() { # Wait for any in-progress deployment wait_for_deployment "$app_name" + # Get current instance count and scale down to 1 to avoid memory quota issues during rolling deploy + echo "Checking current instance count for $app_name..." + local current_instances=$(cf app "$app_name" 2>/dev/null | grep "^instances:" | awk '{print $2}' | cut -d'/' -f2 || echo "1") + echo "Current instances: $current_instances" + + if [ "$current_instances" -gt 1 ]; then + echo "Scaling down to 1 instance to free memory for rolling deploy..." + cf scale "$app_name" -i 1 || true + sleep 5 + fi + for i in $(seq 1 $max_retries); do echo "Attempt $i of $max_retries to push $app_name..." local exit_code=0 @@ -239,6 +250,13 @@ cf_push_with_retry() { echo "Push initiated successfully, waiting for full deployment to complete..." if wait_for_deployment_complete "$app_name"; then echo "Successfully deployed $app_name" + + # Scale back up to original instance count + if [ "$current_instances" -gt 1 ]; then + echo "Scaling back up to $current_instances instances..." + cf scale "$app_name" -i "$current_instances" || true + fi + release_deploy_lock "$app_name" trap - EXIT # Clear the trap return 0 @@ -260,6 +278,13 @@ cf_push_with_retry() { echo "Active deployment detected, waiting for it to complete instead of retrying..." if wait_for_deployment_complete "$app_name"; then echo "Existing deployment completed successfully" + + # Scale back up to original instance count + if [ "$current_instances" -gt 1 ]; then + echo "Scaling back up to $current_instances instances..." + cf scale "$app_name" -i "$current_instances" || true + fi + release_deploy_lock "$app_name" trap - EXIT return 0 @@ -275,6 +300,12 @@ cf_push_with_retry() { fi done + # If we failed, try to scale back up anyway + if [ "$current_instances" -gt 1 ]; then + echo "Deploy failed, attempting to scale back up to $current_instances instances..." + cf scale "$app_name" -i "$current_instances" || true + fi + release_deploy_lock "$app_name" trap - EXIT # Clear the trap echo "Failed to push $app_name after $max_retries attempts" From 988249dbb8ecbed2f23b818b27273f3674520b82 Mon Sep 17 00:00:00 2001 From: Riley Seaburg Date: Wed, 14 Jan 2026 15:10:04 +0000 Subject: [PATCH 2/2] Set SKIP_WIDGET_RENDERER for sidekiq workers to prevent crash Sidekiq workers don't need the Rust widget renderer (it's only used for rendering widgets in the web app). Without SKIP_WIDGET_RENDERER=true, the app crashes on startup because the native library isn't found. This was causing the sidekiq worker to crash with: 'WidgetRenderer native library not found' Now the deploy script sets this env var before pushing. --- .circleci/deploy-sidekiq.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/deploy-sidekiq.sh b/.circleci/deploy-sidekiq.sh index 1db716ff4..3d9338e90 100755 --- a/.circleci/deploy-sidekiq.sh +++ b/.circleci/deploy-sidekiq.sh @@ -126,6 +126,11 @@ cf_push_with_retry() { cf stop "$app_name" || true sleep 5 + # Set SKIP_WIDGET_RENDERER to prevent crash from missing Rust native library + # Sidekiq doesn't need the widget renderer - it's only used for rendering widgets in the web app + echo "Setting SKIP_WIDGET_RENDERER=true for $app_name..." + cf set-env "$app_name" SKIP_WIDGET_RENDERER "true" > /dev/null 2>&1 || true + # Push without rolling strategy (direct replacement since we stopped it) # Let CF auto-detect buildpacks to avoid re-running supply phase (Rust already built in CircleCI) if cf push "$app_name" \