diff --git a/.github/workflows/ftk-pr-cleanup.yml b/.github/workflows/ftk-pr-cleanup.yml new file mode 100644 index 000000000..c7c2a0a0b --- /dev/null +++ b/.github/workflows/ftk-pr-cleanup.yml @@ -0,0 +1,96 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: 'PR Cleanup' + +on: + pull_request_target: + types: [closed] + +permissions: + id-token: write + contents: read + pull-requests: write + issues: write + +jobs: + cleanup: + name: Clean up PR resources + runs-on: ubuntu-latest + environment: ftk-pr + steps: + - name: Install Az modules + shell: pwsh + run: | + Set-PSRepository PSGallery -InstallationPolicy Trusted + Install-Module -Name Az.Accounts -RequiredVersion 2.19.0 -Force + Install-Module -Name Az.Resources -RequiredVersion 6.16.2 -Force + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Delete resource groups + id: delete + shell: pwsh + run: | + $prNumber = "${{ github.event.pull_request.number }}" + $pattern = "pr-$prNumber-*" + $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName -like $pattern } + + if (-not $groups) + { + Write-Host "No resource groups found matching '$pattern'." + "deleted=0" >> $env:GITHUB_OUTPUT + "failed=" >> $env:GITHUB_OUTPUT + return + } + + $deleted = 0 + $failed = @() + foreach ($rg in $groups) + { + try + { + Write-Host "Deleting $($rg.ResourceGroupName)..." + Remove-AzResourceGroup -Name $rg.ResourceGroupName -Force -ErrorAction Stop + $deleted++ + } + catch + { + Write-Warning "Failed to delete $($rg.ResourceGroupName): $_" + $failed += $rg.ResourceGroupName + } + } + + "deleted=$deleted" >> $env:GITHUB_OUTPUT + "failed=$($failed -join ',')" >> $env:GITHUB_OUTPUT + + - name: Post cleanup comment + if: steps.delete.outputs.deleted != '0' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh pr comment "${{ github.event.pull_request.number }}" \ + --repo "${{ github.repository }}" \ + --body "Test environments cleaned up." + + - name: Create issue for failed cleanups + if: steps.delete.outputs.failed != '' + env: + GH_TOKEN: ${{ github.token }} + run: | + failed="${{ steps.delete.outputs.failed }}" + pr=${{ github.event.pull_request.number }} + author=${{ github.event.pull_request.user.login }} + issue_url=$(gh issue create \ + --repo "${{ github.repository }}" \ + --title "Cleanup failed for PR #$pr" \ + --body "The following resource groups could not be deleted: $failed. Please delete them manually. See #$pr.") + # Try to assign the PR author; ignore failure for external contributors + issue_number=$(echo "$issue_url" | grep -oE '[0-9]+$') + gh issue edit "$issue_number" --repo "${{ github.repository }}" --add-assignee "$author" || true diff --git a/.github/workflows/ftk-pr-deploy.yml b/.github/workflows/ftk-pr-deploy.yml new file mode 100644 index 000000000..85ae24da6 --- /dev/null +++ b/.github/workflows/ftk-pr-deploy.yml @@ -0,0 +1,426 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +name: 'PR Deploy' + +on: + pull_request: + types: [opened, synchronize, edited] + paths: + - 'src/templates/**' + - '!src/templates/**/README.md' + - '!src/templates/**/docs/**' + - '!src/templates/**/test/**' + + pull_request_target: + types: [labeled] + +env: + AZ_ACCOUNTS_VERSION: '2.19.0' + AZ_RESOURCES_VERSION: '6.16.2' + AZ_STORAGE_VERSION: '6.2.0' + +concurrency: + group: ftk-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + pull-requests: write + +jobs: + #---------------------------------------------------------------------------- + # Parse PR body checkboxes and determine what to deploy + #---------------------------------------------------------------------------- + check-options: + name: Check deployment options + runs-on: ubuntu-latest + # Skip fork PRs unless they have the deployment label + if: > + (github.event_name == 'pull_request_target' && github.event.label.name == 'Needs: Deployment') + || github.event_name == 'pull_request' + outputs: + deploy-adx: ${{ steps.parse.outputs.deploy-adx }} + deploy-fabric: ${{ steps.parse.outputs.deploy-fabric }} + deploy-manual: ${{ steps.parse.outputs.deploy-manual }} + deploy-nodata: ${{ steps.parse.outputs.deploy-nodata }} + deploy-workbooks: ${{ steps.parse.outputs.deploy-workbooks }} + deploy-alerts: ${{ steps.parse.outputs.deploy-alerts }} + fabric-uri: ${{ steps.parse.outputs.fabric-uri }} + templates-changed: ${{ steps.check-files.outputs.changed }} + steps: + - name: Check for template file changes + id: check-files + env: + GH_TOKEN: ${{ github.token }} + run: | + # For 'edited' events, paths filter doesn't apply — verify manually + changed=$(gh pr diff ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --name-only \ + | grep -E '^src/templates/' \ + | grep -v -E '(README\.md|/docs/|/test/)' \ + | head -1) + if [ -n "$changed" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Verify fork PR safety + if: github.event.pull_request.head.repo.full_name != github.repository + env: + GH_TOKEN: ${{ github.token }} + run: | + # Block fork PRs that modify workflow or script files + modified=$(gh pr diff ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --name-only \ + | grep -E '^(\.github/|src/scripts/)' \ + | head -1) + if [ -n "$modified" ]; then + echo "::error::Fork PRs that modify .github/ or src/scripts/ files cannot use deployment CI." + exit 1 + fi + + - name: Install Az modules + shell: pwsh + run: | + Set-PSRepository PSGallery -InstallationPolicy Trusted + Install-Module -Name Az.Accounts -RequiredVersion $env:AZ_ACCOUNTS_VERSION -Force + Install-Module -Name Az.Resources -RequiredVersion $env:AZ_RESOURCES_VERSION -Force + Install-Module -Name Az.Storage -RequiredVersion $env:AZ_STORAGE_VERSION -Force + + - name: Cache Az modules + uses: actions/cache/save@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + + - name: Parse deployment checkboxes + id: parse + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + parse_checkbox() { + local pattern="$1" + local output_name="$2" + if echo "$PR_BODY" | grep -qE "\- \[x\] $pattern"; then + echo "$output_name=true" >> "$GITHUB_OUTPUT" + return 0 + else + echo "$output_name=false" >> "$GITHUB_OUTPUT" + return 1 + fi + } + + parse_checkbox "Hubs \+ ADX \(managed\)" "deploy-adx" + parse_checkbox "Hubs \+ Fabric \(manual\)" "deploy-fabric" + parse_checkbox "Hubs \(manual\)" "deploy-manual" + parse_checkbox "Hubs \(no data\)" "deploy-nodata" + parse_checkbox "Workbooks" "deploy-workbooks" + parse_checkbox "Alerts" "deploy-alerts" + + # Extract Fabric URI from the checkbox line + fabric_uri=$(echo "$PR_BODY" | grep -oP '(?<=Hubs \+ Fabric \(manual\) — URI: )\S+' || true) + echo "fabric-uri=$fabric_uri" >> "$GITHUB_OUTPUT" + + #---------------------------------------------------------------------------- + # Hub deployments + #---------------------------------------------------------------------------- + # NOTE: The ftk-pr environment MUST have required reviewers configured in GitHub + # settings to gate fork PR deployments. Without this, fork PRs could deploy + # malicious ARM templates using CI credentials. + deploy-adx: + name: 'Hubs + ADX (managed)' + needs: check-options + if: needs.check-options.outputs.deploy-adx == 'true' && needs.check-options.outputs.templates-changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ftk-pr + steps: + - name: Restore Az modules + uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + fail-on-cache-miss: true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy + shell: pwsh + run: | + ./src/scripts/Deploy-Hub -PR ${{ github.event.pull_request.number }} -Name "adx" ` + -Scope "${{ secrets.CI_SCOPE }}" -ManagedExports -Build + + - name: Post PR comment + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + pr=${{ github.event.pull_request.number }} + rg="pr-$pr-adx" + if [ "${{ job.status }}" = "success" ]; then + body="**Hubs + ADX (managed)** deployed to [\`$rg\`](https://portal.azure.com/#@/resource/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$rg/overview)." + else + body="**Hubs + ADX (managed)** deployment failed. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + fi + gh pr comment "$pr" --repo "${{ github.repository }}" --body "$body" + + deploy-fabric: + name: 'Hubs + Fabric (manual)' + needs: check-options + if: needs.check-options.outputs.deploy-fabric == 'true' && needs.check-options.outputs.templates-changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ftk-pr + steps: + - name: Validate Fabric URI + env: + FABRIC_URI: ${{ needs.check-options.outputs.fabric-uri }} + run: | + if [ -z "$FABRIC_URI" ]; then + echo "::error::Hubs + Fabric (manual) checkbox is checked but no URI was provided. Add the eventhouse query URI to the PR body." + exit 1 + fi + + - name: Restore Az modules + uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + fail-on-cache-miss: true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy + shell: pwsh + env: + FABRIC_URI: ${{ needs.check-options.outputs.fabric-uri }} + run: | + ./src/scripts/Deploy-Hub -PR ${{ github.event.pull_request.number }} -Name "fabric" ` + -Fabric "$env:FABRIC_URI" ` + -Scope "${{ secrets.CI_SCOPE }}" -Build + + - name: Post PR comment + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + pr=${{ github.event.pull_request.number }} + rg="pr-$pr-fabric" + if [ "${{ job.status }}" = "success" ]; then + body="**Hubs + Fabric (manual)** deployed to [\`$rg\`](https://portal.azure.com/#@/resource/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$rg/overview). Fabric requires running KQL setup scripts manually." + else + body="**Hubs + Fabric (manual)** deployment failed. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + fi + gh pr comment "$pr" --repo "${{ github.repository }}" --body "$body" + + deploy-manual: + name: 'Hubs (manual)' + needs: check-options + if: needs.check-options.outputs.deploy-manual == 'true' && needs.check-options.outputs.templates-changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ftk-pr + steps: + - name: Restore Az modules + uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + fail-on-cache-miss: true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy + shell: pwsh + run: | + ./src/scripts/Deploy-Hub -PR ${{ github.event.pull_request.number }} -Name "manual" ` + -StorageOnly -Scope "${{ secrets.CI_SCOPE }}" -Build + + - name: Post PR comment + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + pr=${{ github.event.pull_request.number }} + rg="pr-$pr-manual" + if [ "${{ job.status }}" = "success" ]; then + body="**Hubs (manual)** deployed to [\`$rg\`](https://portal.azure.com/#@/resource/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$rg/overview)." + else + body="**Hubs (manual)** deployment failed. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + fi + gh pr comment "$pr" --repo "${{ github.repository }}" --body "$body" + + deploy-nodata: + name: Hubs (no data) + needs: check-options + if: needs.check-options.outputs.deploy-nodata == 'true' && needs.check-options.outputs.templates-changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ftk-pr + steps: + - name: Restore Az modules + uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + fail-on-cache-miss: true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy + shell: pwsh + run: | + ./src/scripts/Deploy-Hub -PR ${{ github.event.pull_request.number }} -Name "nodata" ` + -StorageOnly -Build + + - name: Post PR comment + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + pr=${{ github.event.pull_request.number }} + rg="pr-$pr-nodata" + if [ "${{ job.status }}" = "success" ]; then + body="**Hubs (no data)** deployed to [\`$rg\`](https://portal.azure.com/#@/resource/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$rg/overview)." + else + body="**Hubs (no data)** deployment failed. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + fi + gh pr comment "$pr" --repo "${{ github.repository }}" --body "$body" + + #---------------------------------------------------------------------------- + # Non-hub deployments + #---------------------------------------------------------------------------- + deploy-workbooks: + name: Workbooks + needs: check-options + if: needs.check-options.outputs.deploy-workbooks == 'true' && needs.check-options.outputs.templates-changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ftk-pr + steps: + - name: Restore Az modules + uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + fail-on-cache-miss: true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy + shell: pwsh + run: | + ./src/scripts/Deploy-Toolkit "finops-workbooks" ` + -ResourceGroup "pr-${{ github.event.pull_request.number }}-workbooks" -Build + + - name: Post PR comment + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + pr=${{ github.event.pull_request.number }} + rg="pr-$pr-workbooks" + if [ "${{ job.status }}" = "success" ]; then + body="**Workbooks** deployed to [\`$rg\`](https://portal.azure.com/#@/resource/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$rg/overview)." + else + body="**Workbooks** deployment failed. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + fi + gh pr comment "$pr" --repo "${{ github.repository }}" --body "$body" + + deploy-alerts: + name: Alerts + needs: check-options + if: needs.check-options.outputs.deploy-alerts == 'true' && needs.check-options.outputs.templates-changed == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + environment: ftk-pr + steps: + - name: Restore Az modules + uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/powershell/Modules + key: az-modules-${{ env.AZ_ACCOUNTS_VERSION }}-${{ env.AZ_RESOURCES_VERSION }}-${{ env.AZ_STORAGE_VERSION }} + fail-on-cache-miss: true + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy + shell: pwsh + run: | + ./src/scripts/Deploy-Toolkit "finops-alerts" ` + -ResourceGroup "pr-${{ github.event.pull_request.number }}-alerts" -Build + + - name: Post PR comment + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + pr=${{ github.event.pull_request.number }} + rg="pr-$pr-alerts" + if [ "${{ job.status }}" = "success" ]; then + body="**Alerts** deployed to [\`$rg\`](https://portal.azure.com/#@/resource/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/$rg/overview)." + else + body="**Alerts** deployment failed. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + fi + gh pr comment "$pr" --repo "${{ github.repository }}" --body "$body"