push.yml

name: build

on:
  pull_request:
    types: [opened, synchronize]
  merge_group:
    types: [checks_requested]
  push:
    # Always run on push to main. The build cache can only be reused
    # if it was saved by a run from the repository's default branch.
    # The run result will be identical to that from the merge queue
    # because the commit is identical, yet we need to perform it to
    # seed the build cache.
    branches:
      - main
  schedule:
    - cron: '0 0,12 * * *' # Runs at 00:00 and 12:00 UTC daily

env:
  GOTESTSUM_FORMAT: github-actions

jobs:
  cleanups:
    runs-on:
      group: databricks-deco-testing-runner-group
      labels: ubuntu-latest-deco
    steps:
      - name: Clean up cache if running on schedule
        if: ${{ github.event_name == 'schedule' }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh cache delete --all --repo databricks/cli || true

  testmask:
    runs-on: ubuntu-latest
    outputs:
      targets: ${{ steps.mask1.outputs.targets || steps.mask2.outputs.targets || steps.mask3.outputs.targets }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version-file: tools/go.mod

      - name: Run testmask (pull requests)
        if: ${{ github.event_name == 'pull_request' }}
        id: mask1
        working-directory: tools/testmask
        run: |
          go run . ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.base.sha }} | tee output.json
          echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT

      - name: Run testmask (merge group)
        if: ${{ github.event_name == 'merge_group' }}
        id: mask2
        working-directory: tools/testmask
        run: |
          go run . ${{ github.event.merge_group.head_sha }} ${{ github.event.merge_group.base_sha }} | tee output.json
          echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT

      - name: Run testmask (other events)
        if: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }}
        id: mask3
        working-directory: tools/testmask
        run: |
          # Always run all tests
          echo "targets=[\"test\"]" >> $GITHUB_OUTPUT

  test:
    needs:
      - cleanups
      - testmask

    # Only run if the target is in the list of targets from testmask
    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }}
    name: "task test (${{matrix.os.name}}, ${{matrix.deployment}})"
    runs-on: ${{ matrix.os.runner }}

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: read

    strategy:
      fail-fast: false
      matrix:
        # Use separate fields for the OS name and runner configuration.
        # When combined in a single object, "runs-on" errors with "Unexpected value 'name'".
        os:
          - name: linux
            runner:
              group: databricks-protected-runner-group-large
              labels: linux-ubuntu-latest-large

          - name: windows
            runner:
              group: databricks-protected-runner-group-large
              labels: windows-server-latest-large

          - name: macos
            runner:
              labels: macos-latest

        deployment:
          - "terraform"
          - "direct"

        # Include "event_name" in the matrix so we can include/exclude based on it.
        event:
          - ${{ github.event_name }}

        # Run on Linux only in merge queue to reduce time to merge.
        exclude:
          - event: merge_group
            os:
              name: windows
          - event: merge_group
            os:
              name: macos

    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup build environment
        uses: ./.github/actions/setup-build-environment
        with:
          cache-key: test-${{ matrix.deployment }}

      - name: Run tests without coverage
        # We run tests without coverage on PR, merge_group, and schedule because we don't make use of coverage information
        # and would like to run the tests as fast as possible. We run it on schedule as well, because that is what
        # populates the cache and cache may include test results.
        if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'schedule' }}
        env:
          ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }}
        run: go tool -modfile=tools/task/go.mod task test

      - name: Run tests with coverage
        # Only run 'task cover' on push to main to make sure it does not get broken.
        if: ${{ github.event_name == 'push' }}
        env:
          ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }}
        run: go tool -modfile=tools/task/go.mod task cover

      - name: Analyze slow tests
        run: go tool -modfile=tools/task/go.mod task slowest

      - name: Check out.test.toml files are up to date
        run: |
          if ! git diff --exit-code; then
            echo "ERROR: detected changed files in the repository; Most likely you have out.test.toml files that are out of date. Run 'go test ./acceptance -run \"^TestAccept$\" -only-out-test-toml' to update."
            exit 1
          fi

  test-exp-aitools:
    needs:
      - cleanups
      - testmask

    # Only run if the target is in the list of targets from testmask
    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-aitools') }}
    name: "task test-exp-aitools (${{matrix.os.name}})"
    runs-on: ${{ matrix.os.runner }}

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: read

    strategy:
      fail-fast: false
      matrix:
        os:
          - name: linux
            runner:
              group: databricks-protected-runner-group-large
              labels: linux-ubuntu-latest-large

          - name: macos
            runner:
              labels: macos-latest

        # The Windows tests are broken; see https://github.com/databricks/cli/pull/4024.
        # - name: windows
        #   runner:
        #     group: databricks-protected-runner-group-large
        #     labels: windows-server-latest-large

    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup build environment
        uses: ./.github/actions/setup-build-environment
        with:
          cache-key: test-exp-aitools

      - name: Run tests
        run: |
          go tool -modfile=tools/task/go.mod task test-exp-aitools

  test-exp-ssh:
    needs:
      - cleanups
      - testmask

    # Only run if the target is in the list of targets from testmask
    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }}
    name: "task test-exp-ssh (${{matrix.os.name}})"
    runs-on: ${{ matrix.os.runner }}

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: read

    strategy:
      fail-fast: false
      matrix:
        os:
          - name: linux
            runner:
              group: databricks-protected-runner-group-large
              labels: linux-ubuntu-latest-large

          - name: windows
            runner:
              group: databricks-protected-runner-group-large
              labels: windows-server-latest-large

          - name: macos
            runner:
              labels: macos-latest

    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup build environment
        uses: ./.github/actions/setup-build-environment
        with:
          cache-key: test-exp-ssh

      - name: Run tests
        run: |
          go tool -modfile=tools/task/go.mod task test-exp-ssh

  test-pipelines:
    needs:
      - cleanups
      - testmask

    # Only run if the target is in the list of targets from testmask
    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-pipelines') }}
    name: "task test-pipelines (${{matrix.os.name}})"
    runs-on: ${{ matrix.os.runner }}

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: read

    strategy:
      fail-fast: false
      matrix:
        os:
          - name: linux
            runner:
              group: databricks-protected-runner-group-large
              labels: linux-ubuntu-latest-large

          - name: windows
            runner:
              group: databricks-protected-runner-group-large
              labels: windows-server-latest-large

          - name: macos
            runner:
              labels: macos-latest

    steps:
      - name: Checkout repository and submodules
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup build environment
        uses: ./.github/actions/setup-build-environment
        with:
          cache-key: test-pipelines

      - name: Run tests
        run: |
          go tool -modfile=tools/task/go.mod task test-pipelines

  # This job groups the result of all the above test jobs.
  # It is a required check, so it blocks auto-merge and the merge queue.
  #
  # We use `if: always()` to ensure this job runs even when dependencies are skipped.
  # Without it, GitHub Actions skips jobs whose dependencies are skipped, which would
  # incorrectly block the merge queue when optional test jobs don't run.
  #
  # The step checks `contains(needs.*.result, 'failure')` to fail if any dependency failed.
  # Reference: https://github.com/orgs/community/discussions/25970
  test-result:
    needs:
      - test
      - test-exp-aitools
      - test-exp-ssh
      - test-pipelines

    if: ${{ always() }}
    name: test-result
    runs-on: ubuntu-latest

    steps:
      - run: |
          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
            echo "One or more required jobs failed ❌"
            exit 1
          fi
          echo "All tests passed ✅"

  validate-generated-is-up-to-date:
    needs: cleanups
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version-file: go.mod
          # Use different schema from regular job, to avoid overwriting the same key
          cache-dependency-path: |
            go.sum
            bundle/internal/schema/*.*

      - name: Verify that the schema is up to date
        run: |
          if ! ( go tool -modfile=tools/task/go.mod task --force generate-schema && git diff --exit-code ); then
            echo "The schema is not up to date. Please run './task generate-schema' and commit the changes."
            exit 1
          fi

      - name: Verify that the generated enum and required fields are up to date
        run: |
          if ! ( go tool -modfile=tools/task/go.mod task --force generate-validation && git diff --exit-code ); then
            echo "The generated enum and required fields are not up to date. Please run './task generate-validation' and commit the changes."
            exit 1
          fi

  validate-python-codegen:
    needs: cleanups
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version-file: go.mod

      - name: Install uv
        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
        with:
          version: "0.6.5"

      - name: Verify that python/codegen is up to date
        run: |-
          go tool -modfile=tools/task/go.mod task pydabs-codegen

          if ! ( git diff --exit-code ); then
            echo "Generated Python code is not up-to-date. Please run './task pydabs-codegen' and commit the changes."

            exit 1
          fi

  # Trigger integration tests in a separate repository.
  # Writes the same-org "Integration Tests" check run for skip/auto-approve
  # paths on deco runners. The cross-org `gh workflow run` dispatch is split
  # into the sibling `trigger-tests` job so it can run on emu-access runners
  # that are allowlisted in the databricks-eng org.
  integration-trigger:
    needs:
      - testmask

    if: >-
      (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]') ||
      (github.event_name == 'merge_group')

    runs-on:
      group: databricks-deco-testing-runner-group
      labels: ubuntu-latest-deco

    permissions:
      checks: write
      contents: read

    environment: "test-trigger-is"

    steps:
      - name: Generate GitHub App Token (check runs)
        if: >-
          (github.event_name == 'merge_group') ||
          (github.event_name == 'pull_request' && !contains(fromJSON(needs.testmask.outputs.targets), 'test') && !contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh'))
        id: generate-check-token
        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
        with:
          app-id: ${{ secrets.DECO_TEST_APPROVAL_APP_ID }}
          private-key: ${{ secrets.DECO_TEST_APPROVAL_PRIVATE_KEY }}
          # DECO_TEST_APPROVAL is installed on the databricks org (not databricks-eng).
          owner: databricks
          repositories: cli

      # Skip integration tests if the primary "test" target is not triggered by this change.
      # Use Checks API (not Statuses API) to match the required "Integration Tests" check.
      - name: Skip integration tests (pull request)
        if: ${{ github.event_name == 'pull_request' && !contains(fromJSON(needs.testmask.outputs.targets), 'test') && !contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          github-token: ${{ steps.generate-check-token.outputs.token }}
          script: |
            await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'Integration Tests',
              head_sha: '${{ github.event.pull_request.head.sha }}',
              status: 'completed',
              conclusion: 'success',
              output: {
                title: 'Integration Tests',
                summary: '⏭️ Skipped (changes do not require integration tests)'
              }
            });

      # Auto-approve for merge group since tests already passed on the PR.
      - name: Auto-approve for merge group
        if: ${{ github.event_name == 'merge_group' }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          github-token: ${{ steps.generate-check-token.outputs.token }}
          script: |
            await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'Integration Tests',
              head_sha: context.sha,
              status: 'completed',
              conclusion: 'success',
              output: {
                title: 'Integration Tests',
                summary: '⏭️ Auto-approved for merge queue (tests already passed on PR)'
              }
            });

  # Cross-org dispatch to databricks-eng/eng-dev-ecosystem. Must run on an
  # emu-access runner because the databricks-eng org IP-allowlists only the
  # release runner group, not deco. See databricks/databricks-sdk-go#1638.
  trigger-tests:
    needs:
      - testmask

    if: >-
      (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && (contains(fromJSON(needs.testmask.outputs.targets), 'test') || contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh'))) ||
      (github.event_name == 'push')

    runs-on:
      group: databricks-release-runner-group-emu-access
      labels: linux-ubuntu-latest-emu-access

    permissions:
      contents: read

    environment: "test-trigger-is"

    steps:
      - name: Generate GitHub App Token
        id: generate-token
        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
        with:
          app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }}
          private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }}
          owner: ${{ secrets.ORG_NAME }}
          repositories: ${{ secrets.REPO_NAME }}

      - name: Trigger integration tests (pull request)
        if: ${{ github.event_name == 'pull_request' }}
        env:
          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
        run: |-
          gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{ secrets.REPO_NAME }} \
            --ref main \
            -f pull_request_number=${{ github.event.pull_request.number }} \
            -f commit_sha=${{ github.event.pull_request.head.sha }}

      - name: Trigger integration tests (push to main)
        if: ${{ github.event_name == 'push' }}
        env:
          GH_TOKEN: ${{ steps.generate-token.outputs.token }}
        run: |-
          gh workflow run cli-isolated-nightly.yml -R ${{ secrets.ORG_NAME }}/${{ secrets.REPO_NAME }} \
            --ref main \
            -f commit_sha=${{ github.event.after }}

  # Skip integration tests for dependabot PRs.
  # Dependabot has no access to the "test-trigger-is" environment secrets,
  # so we use the built-in GITHUB_TOKEN to mark the required "Integration
  # Tests" check as passed.
  integration-trigger-dependabot:
    if: >-
      github.event_name == 'pull_request' &&
      github.actor == 'dependabot[bot]'

    runs-on:
      group: databricks-deco-testing-runner-group
      labels: ubuntu-latest-deco

    permissions:
      checks: write

    steps:
      - name: Skip integration tests
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |-
            await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'Integration Tests',
              head_sha: '${{ github.event.pull_request.head.sha }}',
              status: 'completed',
              conclusion: 'success',
              output: {
                title: 'Integration Tests',
                summary: '⏭️ Skipped (dependabot PR)'
              }
            });