diff --git a/.github/actions/resolve-api-meta/action.yml b/.github/actions/resolve-api-meta/action.yml deleted file mode 100644 index 72ef05154..000000000 --- a/.github/actions/resolve-api-meta/action.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Resolve API commit/version -description: Resolve deployed API commit SHA and version from the metadata endpoint -inputs: - api_base_url: - description: Base URL host for the API (e.g. api-dev.mobilitydatabase.org) - required: false - default: api.mobilitydatabase.org - api_refresh_token: - description: API refresh token - required: false -outputs: - COMMIT_SHA: - description: Resolved commit SHA - value: ${{ steps.resolve.outputs.COMMIT_SHA }} - API_VERSION: - description: Resolved API version - value: ${{ steps.resolve.outputs.API_VERSION }} - RESOLVED: - description: Whether a commit SHA was resolved (true/false) - value: ${{ steps.resolve.outputs.RESOLVED }} -runs: - using: composite - steps: - - id: resolve - name: Resolve via API and expose outputs - shell: bash - env: - API_BASE_URL: ${{ inputs.api_base_url }} - API_REFRESH_TOKEN: ${{ inputs.api_refresh_token }} - run: | - # Do not exit on failure; this action should never abort the caller workflow. - set -u - COMMIT_SHA="" - API_VERSION="" - RESOLVED="false" - - if [[ -n "${API_REFRESH_TOKEN:-}" ]]; then - echo "Resolving API commit from https://${API_BASE_URL}/v1/metadata ..." - - # Exchange refresh token -> access token (handle failures gracefully) - REPLY_JSON=$(curl --silent --show-error --location "https://${API_BASE_URL}/v1/tokens" \ - --header 'Content-Type: application/json' \ - --data "{ \"refresh_token\": \"${API_REFRESH_TOKEN}\" }" ) || { - echo "Warning: token exchange failed; will fallback to 'main'" >&2 - REPLY_JSON="" - } - - if [[ -n "${REPLY_JSON}" ]]; then - ACCESS_TOKEN=$(echo "${REPLY_JSON}" | jq -r .access_token 2>/dev/null || echo "") - if [[ -z "${ACCESS_TOKEN}" || "${ACCESS_TOKEN}" == "null" ]]; then - echo "Warning: Could not obtain access token from reply; will fallback to 'main'" >&2 - else - META_JSON=$(curl --silent --show-error \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H 'accept: application/json' \ - "https://${API_BASE_URL}/v1/metadata" ) || { - echo "Warning: metadata request failed; will fallback to 'main'" >&2 - META_JSON="" - } - - if [[ -n "${META_JSON}" ]]; then - COMMIT_SHA=$(echo "${META_JSON}" | jq -r .commit_hash 2>/dev/null || echo "") - API_VERSION=$(echo "${META_JSON}" | jq -r .version 2>/dev/null || echo "") - if [[ -n "${COMMIT_SHA}" && "${COMMIT_SHA}" != "null" ]]; then - RESOLVED="true" - echo "Resolved API version: ${API_VERSION} (commit ${COMMIT_SHA})" - else - echo "Warning: commit_hash missing in metadata; will fallback to 'main'" >&2 - fi - fi - fi - fi - else - echo "No API refresh token provided; skipping API metadata resolution and falling back to 'main'." - fi - - # Expose outputs (empty COMMIT_SHA and RESOLVED=false indicate fallback to 'main') - echo "COMMIT_SHA=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" - echo "API_VERSION=${API_VERSION}" >> "$GITHUB_OUTPUT" - echo "RESOLVED=${RESOLVED}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/db-update-dev.yml b/.github/workflows/db-update-dev.yml index b26cfdc1c..542671489 100644 --- a/.github/workflows/db-update-dev.yml +++ b/.github/workflows/db-update-dev.yml @@ -10,17 +10,6 @@ on: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] workflow_dispatch: - inputs: - DRY_RUN: - description: Dry run. Skip applying schema and content updates - required: false - default: false - type: boolean - INSTALL_LATEST: - description: Install the latest (main) API version when true; when false install the currently deployed version. - required: false - default: false - type: boolean jobs: update: uses: ./.github/workflows/db-update.yml @@ -30,14 +19,6 @@ jobs: DB_NAME: ${{ vars.DEV_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.DEV_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} - API_BASE_URL: api-dev.mobilitydatabase.org - # DRY_RUN is only if requested by the user in a workflow_dispatch - DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} - # We want to use the currently installed version (not the latest) if we received a dispatch from - # mobility_database_catalog. - # For a workflow_dispatch (manual trigger), we use the value set by the user. - INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} - secrets: DB_USER_PASSWORD: ${{ secrets.DEV_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.DEV_POSTGRE_USER_NAME }} @@ -47,7 +28,6 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} - API_TEST_REFRESH_TOKEN: ${{ secrets.DEV_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') diff --git a/.github/workflows/db-update-prod.yml b/.github/workflows/db-update-prod.yml index 2c539965b..cfeff46cb 100644 --- a/.github/workflows/db-update-prod.yml +++ b/.github/workflows/db-update-prod.yml @@ -1,22 +1,10 @@ # Update the Mobility Database Schema name: Database Update - PROD on: - workflow_dispatch: # Manual trigger - inputs: - DRY_RUN: - description: Dry run. Skip applying schema and content updates - required: false - default: false - type: boolean - INSTALL_LATEST: - description: Install the latest (main) API version when true; when false install the currently deployed version. - required: false - default: false - type: boolean + workflow_dispatch: workflow_call: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] - jobs: update: uses: ./.github/workflows/db-update.yml @@ -26,16 +14,6 @@ jobs: DB_NAME: ${{ vars.PROD_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.PROD_MOBILITY_FEEDS_ENVIRONMENT }} - API_BASE_URL: api.mobilitydatabase.org - # DRY_RUN is only if requested by the user in a workflow_dispatch - DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} - # We want to use the currently installed version (not the latest) if we received a dispatch from - # mobility_database_catalog. - # We want to use the latest version if the trigger was workflow_call, because currently it's used for - # upgrading (e.g. release.yml) - # For a workflow_dispatch (manual trigger), we use the value set by the user. - INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} - secrets: DB_USER_PASSWORD: ${{ secrets.PROD_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.PROD_POSTGRE_USER_NAME }} @@ -45,7 +23,6 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} - API_TEST_REFRESH_TOKEN: ${{ secrets.PROD_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] diff --git a/.github/workflows/db-update-qa.yml b/.github/workflows/db-update-qa.yml index 78a3084d7..62262e68c 100644 --- a/.github/workflows/db-update-qa.yml +++ b/.github/workflows/db-update-qa.yml @@ -2,17 +2,6 @@ name: Database Update - QA on: workflow_dispatch: - inputs: - DRY_RUN: - description: Dry run. Skip applying schema and content updates - required: false - default: false - type: boolean - INSTALL_LATEST: - description: Install the latest (main) API version when true; when false install the currently deployed version. - required: false - default: false - type: boolean workflow_call: repository_dispatch: # Update on mobility-database-catalog repo dispatch types: [ catalog-sources-updated, gbfs-systems-updated ] @@ -26,16 +15,6 @@ jobs: DB_NAME: ${{ vars.QA_POSTGRE_SQL_DB_NAME }} ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} DB_ENVIRONMENT: ${{ vars.QA_MOBILITY_FEEDS_ENVIRONMENT }} - API_BASE_URL: api-qa.mobilitydatabase.org - # DRY_RUN is only if requested by the user in a workflow_dispatch - DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.DRY_RUN }} - # We want to use the currently installed version (not the latest) if we received a dispatch from - # mobility_database_catalog. - # We want to use the latest version if the trigger was workflow_call, because currently it's used for - # upgrading (e.g. release.yml) - # For a workflow_dispatch (manual trigger), we use the value set by the user. - INSTALL_LATEST: ${{ (github.event_name == 'repository_dispatch' && false) || (github.event_name == 'workflow_call' && true) || (github.event_name == 'workflow_dispatch' && inputs.INSTALL_LATEST) }} - secrets: DB_USER_PASSWORD: ${{ secrets.QA_POSTGRE_USER_PASSWORD }} DB_USER_NAME: ${{ secrets.QA_POSTGRE_USER_NAME }} @@ -45,7 +24,6 @@ jobs: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }} POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }} - API_TEST_REFRESH_TOKEN: ${{ secrets.QA_API_TEST_REFRESH_TOKEN }} notify-slack-on-failure: needs: [ update ] if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch') diff --git a/.github/workflows/db-update.yml b/.github/workflows/db-update.yml index 584cd9729..3ce05c37a 100644 --- a/.github/workflows/db-update.yml +++ b/.github/workflows/db-update.yml @@ -46,9 +46,6 @@ on: POSTGRE_SQL_INSTANCE_NAME: description: PostgreSQL Instance Name required: true - API_TEST_REFRESH_TOKEN: - description: API refresh token used to resolve deployed API commit (used on repository_dispatch) - required: false inputs: PROJECT_ID: description: GCP Project ID @@ -70,61 +67,19 @@ on: description: GCP region required: true type: string - API_BASE_URL: - description: Base URL host for the API used to resolve version/commit (e.g. api-dev.mobilitydatabase.org) - required: false - default: api.mobilitydatabase.org - type: string - INSTALL_LATEST: - description: Install the latest (main) API version when true; when false keep the currently deployed version. - required: false - default: false - type: boolean - DRY_RUN: - description: Skip applying schema and content updates - required: false - default: false - type: boolean env: python_version: '3.11' liquibase_version: '4.33.0' jobs: - resolve-api-meta: - name: 'Resolve API commit/version' - runs-on: ubuntu-latest - # Run this job for all triggers; the action itself will skip resolution when API_BASE_URL or token is not provided. - # Keeping it unconditional ensures CHECKOUT_REF is always set (defaults to 'main') for downstream jobs. - outputs: - # Use resolved commit when available; otherwise default to 'main'. - CHECKOUT_REF: ${{ steps.resolve.outputs.COMMIT_SHA != '' && steps.resolve.outputs.COMMIT_SHA || 'main' }} - steps: - - name: Checkout repo (for scripts and local action) - uses: actions/checkout@v4 - - name: Resolve API commit/version - id: resolve - if: ${{ inputs.INSTALL_LATEST == false }} - uses: ./.github/actions/resolve-api-meta - with: - api_base_url: ${{ inputs.API_BASE_URL }} - api_refresh_token: ${{ secrets.API_TEST_REFRESH_TOKEN }} - db-schema-update: name: 'Database Schema Update' permissions: write-all runs-on: ubuntu-latest - needs: [resolve-api-meta] - # Run the schema update when the resolved checkout target is 'main' (install latest/main). - # This covers both explicit INSTALL_LATEST runs and cases where resolution failed and CHECKOUT_REF fell back to 'main'. - if: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF == 'main' }} steps: - - name: Checkout repo + - name: Checkout code uses: actions/checkout@v4 - with: - # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') - ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} - fetch-depth: 0 - name: Authenticate to Google Cloud QA/PROD uses: google-github-actions/auth@v2 @@ -171,7 +126,6 @@ jobs: liquibase --version - name: Run Liquibase - if: ${{ !inputs.DRY_RUN }} working-directory: ${{ github.workspace }}/liquibase run: | export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml" @@ -185,15 +139,11 @@ jobs: name: 'Database Content Update' permissions: write-all runs-on: ubuntu-latest - needs: [resolve-api-meta, db-schema-update] - if: ${{ always() }} + needs: db-schema-update + if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} steps: - - name: Checkout repo + - name: Checkout code uses: actions/checkout@v4 - with: - # Use the job-level CHECKOUT_REF (already resolves to COMMIT_SHA or 'main') - ref: ${{ needs.resolve-api-meta.outputs.CHECKOUT_REF }} - fetch-depth: 0 - name: Setup python uses: actions/setup-python@v5 @@ -262,11 +212,11 @@ jobs: run: echo "PATH=$(realpath sources.csv)" >> $GITHUB_OUTPUT - name: GTFS - Update Database Content - if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual' }} run: scripts/populate-db.sh ${{ steps.getpath.outputs.PATH }} > populate.log - name: GTFS - Upload log file for verification - if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ always() && (env.UPDATE_TYPE == 'gtfs' || env.UPDATE_TYPE == 'manual') }} uses: actions/upload-artifact@v4 with: name: populate-${{ inputs.ENVIRONMENT }}.log @@ -282,11 +232,11 @@ jobs: run: echo "PATH=$(realpath systems.csv)" >> $GITHUB_OUTPUT - name: GBFS - Update Database Content - if: ${{ !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual' }} run: scripts/populate-db.sh ${{ steps.getsyspath.outputs.PATH }} gbfs >> populate-gbfs.log - name: GBFS - Upload log file for verification - if: ${{ always() && !inputs.DRY_RUN && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} + if: ${{ always() && (env.UPDATE_TYPE == 'gbfs' || env.UPDATE_TYPE == 'manual') }} uses: actions/upload-artifact@v4 with: name: populate-gbfs-${{ inputs.ENVIRONMENT }}.log @@ -295,7 +245,7 @@ jobs: update-gcp-secret: name: Update GCP Secrets - if: ${{ contains('repository_dispatch,workflow_dispatch', github.event_name) && !inputs.DRY_RUN }} + if: ${{ github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' }} runs-on: ubuntu-latest steps: - name: Authenticate to Google Cloud @@ -333,3 +283,6 @@ jobs: echo "Secret $SECRET_NAME does not exist in project $PROJECT_ID, creating..." echo -n "$SECRET_VALUE" | gcloud secrets create $SECRET_NAME --data-file=- --replication-policy="automatic" --project=$PROJECT_ID fi + + +