Scheduled - Test All Features #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Scheduled - Test All Features" | |
| on: | |
| schedule: | |
| - cron: '0 0 * * 0' # Every Sunday at midnight UTC | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| get-all-features: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| features: ${{ steps.list.outputs.features }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "List all features" | |
| id: list | |
| run: | | |
| FEATURES=$(ls src/ | sort | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "features=$FEATURES" >> $GITHUB_OUTPUT | |
| test-autogenerated: | |
| needs: get-all-features | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 10 | |
| matrix: | |
| feature: ${{ fromJson(needs.get-all-features.outputs.features) }} | |
| baseImage: | |
| - debian:latest | |
| - ubuntu:latest | |
| - mcr.microsoft.com/devcontainers/base:ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Sanitize base image name" | |
| run: | | |
| BASE_IMAGE_SAFE=$(echo "${{ matrix.baseImage }}" | sed 's/[/: ]/-/g') | |
| echo "BASE_IMAGE_SAFE=$BASE_IMAGE_SAFE" >> $GITHUB_ENV | |
| - name: "Install latest devcontainer CLI" | |
| run: npm install -g @devcontainers/cli | |
| - name: "Test '${{ matrix.feature }}' against '${{ matrix.baseImage }}' (with retry)" | |
| run: | | |
| LOG_FILE="${{ runner.temp }}/test-log.txt" | |
| max_attempts=3 | |
| attempt=1 | |
| while true; do | |
| echo "=== Attempt $attempt of $max_attempts ===" | tee -a "$LOG_FILE" | |
| if devcontainer features test \ | |
| --skip-scenarios \ | |
| -f "${{ matrix.feature }}" \ | |
| -i "${{ matrix.baseImage }}" \ | |
| . 2>&1 | tee -a "$LOG_FILE"; then | |
| echo "Tests passed on attempt $attempt" | |
| exit 0 | |
| fi | |
| echo "Attempt $attempt failed" | tee -a "$LOG_FILE" | |
| if [ "$attempt" -ge "$max_attempts" ]; then | |
| echo "All $max_attempts attempts failed" | tee -a "$LOG_FILE" | |
| exit 1 | |
| fi | |
| attempt=$((attempt + 1)) | |
| echo "Retrying in 30 seconds..." | tee -a "$LOG_FILE" | |
| sleep 30 | |
| done | |
| - name: "Upload failure log" | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: failure-autogen-${{ matrix.feature }}-${{ env.BASE_IMAGE_SAFE }} | |
| path: ${{ runner.temp }}/test-log.txt | |
| test-scenarios: | |
| needs: get-all-features | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 10 | |
| matrix: | |
| feature: ${{ fromJson(needs.get-all-features.outputs.features) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Install latest devcontainer CLI" | |
| run: npm install -g @devcontainers/cli | |
| - name: "Test '${{ matrix.feature }}' scenarios (with retry)" | |
| run: | | |
| LOG_FILE="${{ runner.temp }}/test-log.txt" | |
| max_attempts=3 | |
| attempt=1 | |
| while true; do | |
| echo "=== Attempt $attempt of $max_attempts ===" | tee -a "$LOG_FILE" | |
| if devcontainer features test \ | |
| -f "${{ matrix.feature }}" \ | |
| --skip-autogenerated \ | |
| --skip-duplicated \ | |
| . 2>&1 | tee -a "$LOG_FILE"; then | |
| echo "Tests passed on attempt $attempt" | |
| exit 0 | |
| fi | |
| echo "Attempt $attempt failed" | tee -a "$LOG_FILE" | |
| if [ "$attempt" -ge "$max_attempts" ]; then | |
| echo "All $max_attempts attempts failed" | tee -a "$LOG_FILE" | |
| exit 1 | |
| fi | |
| attempt=$((attempt + 1)) | |
| echo "Retrying in 30 seconds..." | tee -a "$LOG_FILE" | |
| sleep 30 | |
| done | |
| - name: "Upload failure log" | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: failure-scenarios-${{ matrix.feature }} | |
| path: ${{ runner.temp }}/test-log.txt | |
| create-issue: | |
| needs: [test-autogenerated, test-scenarios] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: "Download all failure artifacts" | |
| id: download | |
| continue-on-error: true | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: failure-* | |
| path: ${{ runner.temp }}/failures | |
| - name: "Create issue for failures" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| FAILURES_DIR="${{ runner.temp }}/failures" | |
| if [ ! -d "$FAILURES_DIR" ] || [ -z "$(ls -A "$FAILURES_DIR" 2>/dev/null)" ]; then | |
| echo "No test failures found. Skipping issue creation." | |
| exit 0 | |
| fi | |
| # Build issue body | |
| { | |
| echo "## Scheduled Feature Test Failures" | |
| echo "" | |
| echo "The weekly scheduled test detected failures in one or more features after **3 retry attempts**." | |
| echo "" | |
| echo "**Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| echo "" | |
| echo "---" | |
| echo "" | |
| for dir in "$FAILURES_DIR"/*/; do | |
| artifact_name=$(basename "$dir") | |
| echo "### \`$artifact_name\`" | |
| echo "" | |
| echo "<details>" | |
| echo "<summary>Expand log</summary>" | |
| echo "" | |
| echo '```' | |
| cat "$dir/test-log.txt" 2>/dev/null || echo "No log available" | |
| echo '```' | |
| echo "" | |
| echo "</details>" | |
| echo "" | |
| done | |
| } > "${{ runner.temp }}/issue-body.md" | |
| gh issue create \ | |
| --repo "${{ github.repository }}" \ | |
| --title "Scheduled Test Failures - ${{ github.run_started_at }}" \ | |
| --body-file "${{ runner.temp }}/issue-body.md" |