From 680f73c3a1110e49363237742145060a8018ab0d Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 19 Sep 2025 22:47:07 -0400 Subject: [PATCH 01/16] start over --- .github/workflows/auto-cache/action.yaml | 58 ---- .github/workflows/auto_pr_review.yaml | 52 ---- .github/workflows/badges.yaml | 37 --- .github/workflows/ci_weekly_report.yaml | 101 ------- .github/workflows/ci_weekly_run.yaml | 17 -- .../workflows/compile-openpilot/action.yaml | 21 -- .github/workflows/docs.yaml | 65 ---- .github/workflows/jenkins-pr-trigger.yaml | 59 ---- .github/workflows/model_review.yaml | 42 --- .github/workflows/prebuilt.yaml | 39 --- .github/workflows/release.yaml | 42 --- .github/workflows/repo-maintenance.yaml | 72 ----- .github/workflows/selfdrive_tests.yaml | 279 ------------------ .../workflows/setup-with-retry/action.yaml | 52 ---- .github/workflows/setup/action.yaml | 56 ---- .github/workflows/stale.yaml | 52 ---- .github/workflows/ui_preview.yaml | 174 ----------- 17 files changed, 1218 deletions(-) delete mode 100644 .github/workflows/auto-cache/action.yaml delete mode 100644 .github/workflows/auto_pr_review.yaml delete mode 100644 .github/workflows/badges.yaml delete mode 100644 .github/workflows/ci_weekly_report.yaml delete mode 100644 .github/workflows/ci_weekly_run.yaml delete mode 100644 .github/workflows/compile-openpilot/action.yaml delete mode 100644 .github/workflows/docs.yaml delete mode 100644 .github/workflows/jenkins-pr-trigger.yaml delete mode 100644 .github/workflows/model_review.yaml delete mode 100644 .github/workflows/prebuilt.yaml delete mode 100644 .github/workflows/release.yaml delete mode 100644 .github/workflows/repo-maintenance.yaml delete mode 100644 .github/workflows/selfdrive_tests.yaml delete mode 100644 .github/workflows/setup-with-retry/action.yaml delete mode 100644 .github/workflows/setup/action.yaml delete mode 100644 .github/workflows/stale.yaml delete mode 100644 .github/workflows/ui_preview.yaml diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml deleted file mode 100644 index 377b1eedcde4bb..00000000000000 --- a/.github/workflows/auto-cache/action.yaml +++ /dev/null @@ -1,58 +0,0 @@ -name: 'automatically cache based on current runner' - -inputs: - path: - description: 'path to cache' - required: true - key: - description: 'key' - required: true - restore-keys: - description: 'restore-keys' - required: true - save: - description: 'whether to save the cache' - default: 'true' - required: false -outputs: - cache-hit: - description: 'cache hit occurred' - value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) || - (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) || - (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }} - -runs: - using: "composite" - steps: - - name: setup namespace cache - id: ns-cache - if: ${{ contains(runner.name, 'nsc') }} - uses: namespacelabs/nscloud-cache-action@v1 - with: - path: ${{ inputs.path }} - - - name: setup github cache - id: gha-cache - if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }} - uses: 'actions/cache@v4' - with: - path: ${{ inputs.path }} - key: ${{ inputs.key }} - restore-keys: ${{ inputs.restore-keys }} - - - name: setup github cache - id: gha-cache-ro - if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }} - uses: 'actions/cache/restore@v4' - with: - path: ${{ inputs.path }} - key: ${{ inputs.key }} - restore-keys: ${{ inputs.restore-keys }} - - # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps - - id: scons-cache-setup - shell: bash - run: | - mkdir -p ${{ inputs.path }} - sudo chmod -R 777 ${{ inputs.path }} - sudo chown -R $USER ${{ inputs.path }} diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml deleted file mode 100644 index c6a1cb98219a68..00000000000000 --- a/.github/workflows/auto_pr_review.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: "PR review" -on: - pull_request_target: - types: [opened, reopened, synchronize, edited] - -jobs: - labeler: - name: review - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: false - - # Label PRs - - uses: actions/labeler@v5.0.0 - with: - dot: true - configuration-path: .github/labeler.yaml - - # Check PR target branch - - name: check branch - uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5 - if: github.repository == 'commaai/openpilot' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - target: /^(?!master$).*/ - exclude: /commaai:.*/ - change-to: ${{ github.base_ref }} - already-exists-action: close_this - already-exists-comment: "Your PR should be made against the `master` branch" - - # Welcome comment - - name: "First timers PR" - uses: actions/first-interaction@v1 - if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - pr-message: | - - Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following: - * Convert your PR to a draft unless it's ready to review - * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md) - * Before marking as "ready for review", ensure: - * the goal is clearly stated in the description - * all the tests are passing - * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged) - * include a route or your device' dongle ID if relevant diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml deleted file mode 100644 index 63ee736dcab84b..00000000000000 --- a/.github/workflows/badges.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: badges -on: - schedule: - - cron: '0 * * * *' - workflow_dispatch: - -env: - BASE_IMAGE: openpilot-base - DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c - -jobs: - badges: - name: create badges - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Push badges - run: | - ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py" - - rm .gitattributes - - git checkout --orphan badges - git rm -rf --cached . - git config user.email "badge-researcher@comma.ai" - git config user.name "Badge Researcher" - - git add translation_badge.svg - git commit -m "Add/Update badges" - git push -f origin HEAD diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml deleted file mode 100644 index 9821283cb57442..00000000000000 --- a/.github/workflows/ci_weekly_report.yaml +++ /dev/null @@ -1,101 +0,0 @@ -name: weekly CI test report -on: - schedule: - - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday - workflow_dispatch: - inputs: - ci_runs: - description: 'The amount of runs to trigger in CI test report' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }} - -jobs: - setup: - if: github.repository == 'commaai/openpilot' - runs-on: ubuntu-latest - outputs: - ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }} - steps: - - id: ci_runs_setup - name: CI_RUNS=${{ env.CI_RUNS }} - run: | - matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))") - echo "matrix=$matrix" >> $GITHUB_OUTPUT - - ci_matrix_run: - needs: [ setup ] - strategy: - fail-fast: false - matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}} - uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master - with: - run_number: ${{ matrix.run_number }} - - report: - needs: [ci_matrix_run] - runs-on: ubuntu-latest - if: always() - steps: - - name: Get job results - uses: actions/github-script@v7 - id: get-job-results - with: - script: | - const jobs = await github - .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", { - owner: "commaai", - repo: "${{ github.event.repository.name }}", - run_id: "${{ github.run_id }}", - attempt: "${{ github.run_attempt }}", - }) - var report = {} - jobs.slice(1, jobs.length-1).forEach(job => { - if (job.conclusion === "skipped") return; - const jobName = job.name.split(" / ")[2]; - const runRegex = /\((.*?)\)/; - const run = job.name.match(runRegex)[1]; - report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] }; - switch (job.conclusion) { - case "success": - report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break; - case "failure": - report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break; - case "canceled": - report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break; - } - }); - return JSON.stringify({"jobs": report}); - - - name: Add job results to summary - env: - JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }} - run: | - cat <> template.html - - - - - - - - - - - {% for key in jobs.keys() %} - - - - - - {% endfor %} -
Job✅ Passing❌ Failure Details
{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}{{ key }}{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%{% if jobs[key]["failures"]|length > 0 %}
{% for failure in jobs[key]["failures"] %}Log for run #{{ failure['run_number'] }}
{% endfor %}
{% else %}{% endif %}
- EOF - - pip install jinja2-cli - echo $JOB_RESULTS | jinja2 template.html > report.html - echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY - cat report.html >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/ci_weekly_run.yaml b/.github/workflows/ci_weekly_run.yaml deleted file mode 100644 index d8bf91116d794f..00000000000000 --- a/.github/workflows/ci_weekly_run.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: weekly CI test run -on: - workflow_call: - inputs: - run_number: - required: true - type: string - -concurrency: - group: ci-run-${{ inputs.run_number }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - selfdrive_tests: - uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master - with: - run_number: ${{ inputs.run_number }} diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml deleted file mode 100644 index 4015746c0e3680..00000000000000 --- a/.github/workflows/compile-openpilot/action.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: 'compile openpilot' - -runs: - using: "composite" - steps: - - shell: bash - name: Build openpilot with all flags - run: | - ${{ env.RUN }} "scons -j$(nproc)" - ${{ env.RUN }} "release/check-dirty.sh" - - shell: bash - name: Cleanup scons cache and rebuild - run: | - ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ - scons -j$(nproc) --cache-populate" - - name: Save scons cache - uses: actions/cache/save@v4 - if: github.ref == 'refs/heads/master' - with: - path: .ci_cache/scons_cache - key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml deleted file mode 100644 index 92c311829c0654..00000000000000 --- a/.github/workflows/docs.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: docs - -on: - push: - branches: - - master - pull_request: - workflow_call: - inputs: - run_number: - default: '1' - required: true - type: string -concurrency: - group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - docs: - name: build docs - runs-on: ubuntu-24.04 - steps: - - uses: commaai/timeout@v1 - - - uses: actions/checkout@v4 - with: - submodules: true - - # Build - - name: Build docs - run: | - # TODO: can we install just the "docs" dependency group without the normal deps? - pip install mkdocs - mkdocs build - - # Push to docs.comma.ai - - uses: actions/checkout@v4 - if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' - with: - path: openpilot-docs - ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }} - repository: commaai/openpilot-docs - - name: Push - if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' - run: | - set -x - - source release/identity.sh - - cd openpilot-docs - git checkout --orphan tmp - git rm -rf . - - # copy over docs - cp -r ../docs_site/ docs/ - - # GitHub pages config - touch docs/.nojekyll - echo -n docs.comma.ai > docs/CNAME - - git add -f . - git commit -m "build docs" - - # docs live in different repo to not bloat openpilot's full clone size - git push -f origin tmp:gh-pages diff --git a/.github/workflows/jenkins-pr-trigger.yaml b/.github/workflows/jenkins-pr-trigger.yaml deleted file mode 100644 index 14e2fdf49ba970..00000000000000 --- a/.github/workflows/jenkins-pr-trigger.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: jenkins scan - -on: - issue_comment: - types: [created, edited] - -jobs: - # TODO: gc old branches in a separate job in this workflow - scan-comments: - runs-on: ubuntu-latest - if: ${{ github.event.issue.pull_request }} - permissions: - contents: write - issues: write - steps: - - name: Check for trigger phrase - id: check_comment - uses: actions/github-script@v7 - with: - script: | - const triggerPhrase = "trigger-jenkins"; - const comment = context.payload.comment.body; - const commenter = context.payload.comment.user.login; - - const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: commenter - }); - - const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin'; - - return (hasWriteAccess && comment.includes(triggerPhrase)); - result-encoding: json - - - name: Checkout repository - if: steps.check_comment.outputs.result == 'true' - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ github.event.issue.number }}/head - - - name: Push to tmp-jenkins branch - if: steps.check_comment.outputs.result == 'true' - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b tmp-jenkins-${{ github.event.issue.number }} - GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }} - - - name: Delete trigger comment - if: steps.check_comment.outputs.result == 'true' && always() - uses: actions/github-script@v7 - with: - script: | - await github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: context.payload.comment.id, - }); diff --git a/.github/workflows/model_review.yaml b/.github/workflows/model_review.yaml deleted file mode 100644 index 0e1825864c278c..00000000000000 --- a/.github/workflows/model_review.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "model review" - -on: - pull_request: - types: [opened, reopened, synchronize] - paths: - - 'selfdrive/modeld/models/*.onnx' - workflow_dispatch: - -jobs: - comment: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Checkout master - uses: actions/checkout@v4 - with: - ref: master - path: base - - run: git lfs pull - - run: cd base && git lfs pull - - - run: pip install onnx - - - name: scripts/reporter.py - id: report - run: | - echo "content<> $GITHUB_OUTPUT - echo "## Model Review" >> $GITHUB_OUTPUT - MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Post model report comment - uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16 - with: - header: model-review - message: ${{ steps.report.outputs.content }} \ No newline at end of file diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml deleted file mode 100644 index d8963ec89f4f93..00000000000000 --- a/.github/workflows/prebuilt.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: prebuilt -on: - schedule: - - cron: '0 * * * *' - workflow_dispatch: - -env: - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: selfdrive/test/docker_build.sh prebuilt - -jobs: - build_prebuilt: - name: build prebuilt - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - env: - PUSH_IMAGE: true - permissions: - checks: read - contents: read - packages: write - steps: - - name: Wait for green check mark - if: ${{ github.event_name != 'workflow_dispatch' }} - uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc - with: - ref: master - wait-interval: 30 - running-workflow-name: 'build prebuilt' - repo-token: ${{ secrets.GITHUB_TOKEN }} - check-regexp: ^((?!.*(build master-ci).*).)*$ - - uses: actions/checkout@v4 - with: - submodules: true - - run: git lfs pull - - name: Build and Push docker image - run: | - $DOCKER_LOGIN - eval "$BUILD" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 0f4ce6cb3a718e..00000000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: release -on: - schedule: - - cron: '0 9 * * *' - workflow_dispatch: - -jobs: - build_masterci: - name: build master-ci - env: - ImageOS: ubuntu24 - container: - image: ghcr.io/commaai/openpilot-base:latest - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - permissions: - checks: read - contents: write - steps: - - name: Install wait-on-check-action dependencies - run: | - sudo apt-get update - sudo apt-get install -y libyaml-dev - - name: Wait for green check mark - if: ${{ github.event_name == 'schedule' }} - uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc - with: - ref: master - wait-interval: 30 - running-workflow-name: 'build master-ci' - repo-token: ${{ secrets.GITHUB_TOKEN }} - check-regexp: ^((?!.*(build prebuilt).*).)*$ - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - name: Pull LFS - run: | - git config --global --add safe.directory '*' - git lfs pull - - name: Push master-ci - run: BRANCH=__nightly release/build_stripped.sh diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml deleted file mode 100644 index 7bb91c0ca4fe04..00000000000000 --- a/.github/workflows/repo-maintenance.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: repo maintenance - -on: - schedule: - - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST) - workflow_dispatch: - -env: - BASE_IMAGE: openpilot-base - BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - -jobs: - update_translations: - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - steps: - - uses: actions/checkout@v4 - - uses: ./.github/workflows/setup-with-retry - - name: Update translations - run: | - ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish" - - name: Create Pull Request - uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 - with: - author: Vehicle Researcher - commit-message: "Update translations" - title: "[bot] Update translations" - body: "Automatic PR from repo-maintenance -> update_translations" - branch: "update-translations" - base: "master" - delete-branch: true - labels: bot - - package_updates: - name: package_updates - runs-on: ubuntu-latest - container: - image: ghcr.io/commaai/openpilot-base:latest - if: github.repository == 'commaai/openpilot' - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: uv lock - run: | - python3 -m ensurepip --upgrade - pip3 install uv - uv lock --upgrade - - name: bump submodules - run: | - git config --global --add safe.directory '*' - git submodule update --remote - git add . - - name: update car docs - run: | - export PYTHONPATH="$PWD" - scons -j$(nproc) --minimal opendbc_repo - python selfdrive/car/docs.py - git add docs/CARS.md - - name: Create Pull Request - uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 - with: - author: Vehicle Researcher - token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} - commit-message: Update Python packages - title: '[bot] Update Python packages' - branch: auto-package-updates - base: master - delete-branch: true - body: 'Automatic PR from repo-maintenance -> package_updates' - labels: bot diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml deleted file mode 100644 index beb426c669719b..00000000000000 --- a/.github/workflows/selfdrive_tests.yaml +++ /dev/null @@ -1,279 +0,0 @@ -name: selfdrive - -on: - push: - branches: - - master - pull_request: - workflow_dispatch: - workflow_call: - inputs: - run_number: - default: '1' - required: true - type: string - -concurrency: - group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} - cancel-in-progress: true - -env: - PYTHONWARNINGS: error - BASE_IMAGE: openpilot-base - AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} - - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: selfdrive/test/docker_build.sh base - - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - - PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical - -jobs: - build_release: - name: build release - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - env: - STRIPPED_DIR: /tmp/releasepilot - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Getting LFS files - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e - with: - timeout_minutes: 2 - max_attempts: 3 - command: git lfs pull - - name: Build devel - timeout-minutes: 1 - run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh - - uses: ./.github/workflows/setup-with-retry - - name: Build openpilot and run checks - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "python3 system/manager/build.py" - - name: Run tests - timeout-minutes: 1 - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "release/check-dirty.sh" - - name: Check submodules - if: github.repository == 'commaai/openpilot' - timeout-minutes: 3 - run: release/check-submodules.sh - - build: - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup docker push - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' - run: | - echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - $DOCKER_LOGIN - - uses: ./.github/workflows/setup-with-retry - - uses: ./.github/workflows/compile-openpilot - timeout-minutes: 30 - - build_mac: - name: build macOS - runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV - - name: Homebrew cache - uses: ./.github/workflows/auto-cache - with: - path: ~/Library/Caches/Homebrew - key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} - restore-keys: | - brew-macos-${{ env.CACHE_COMMIT_DATE }} - brew-macos - - name: Install dependencies - run: ./tools/mac_setup.sh - env: - PYTHONWARNINGS: default # package install has DeprecationWarnings - HOMEBREW_DISPLAY_INSTALL_TIMES: 1 - - run: git lfs pull - - name: Getting scons cache - uses: ./.github/workflows/auto-cache - with: - path: /tmp/scons_cache - key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} - restore-keys: | - scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }} - scons-${{ runner.arch }}-macos - - name: Building openpilot - run: . .venv/bin/activate && scons -j$(nproc) - - static_analysis: - name: static analysis - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - env: - PYTHONWARNINGS: default - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Static analysis - timeout-minutes: 1 - run: ${{ env.RUN }} "scripts/lint/lint.sh" - - unit_tests: - name: unit tests - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - id: setup-step - - name: Build openpilot - run: ${{ env.RUN }} "scons -j$(nproc)" - - name: Run unit tests - timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} - run: | - ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ - # Pre-compile Python bytecode so each pytest worker doesn't need to - $PYTEST --collect-only -m 'not slow' -qq && \ - MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \ - ./selfdrive/ui/tests/create_test_translations.sh && \ - QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ - chmod -R 777 /tmp/comma_download_cache" - - process_replay: - name: process replay - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - id: setup-step - - name: Cache test routes - id: dependency-cache - uses: actions/cache@v4 - with: - path: .ci_cache/comma_download_cache - key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }} - - name: Build openpilot - run: | - ${{ env.RUN }} "scons -j$(nproc)" - - name: Run replay - timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} - run: | - ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ - chmod -R 777 /tmp/comma_download_cache" - - name: Print diff - id: print-diff - if: always() - run: cat selfdrive/test/process_replay/diff.txt - - uses: actions/upload-artifact@v4 - if: always() - continue-on-error: true - with: - name: process_replay_diff.txt - path: selfdrive/test/process_replay/diff.txt - - name: Upload reference logs - if: false # TODO: move this to github instead of azure - run: | - ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only" - - name: Run regen - if: false - timeout-minutes: 4 - run: | - ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ - chmod -R 777 /tmp/comma_download_cache" - - simulator_driving: - name: simulator driving - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - id: setup-step - - name: Build openpilot - run: | - ${{ env.RUN }} "scons -j$(nproc)" - - name: Driving test - timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }} - run: | - ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ - source selfdrive/test/setup_vsound.sh && \ - CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py" - - create_ui_report: - # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml - name: Create UI Report - runs-on: ${{ - (github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) - && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') - || fromJSON('["ubuntu-24.04"]') }} - if: false # FIXME: FrameReader is broken on CI runners - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: caching frames - id: frames-cache - uses: actions/cache@v4 - with: - path: .ci_cache/comma_download_cache - key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }} - - name: Build openpilot - run: ${{ env.RUN }} "scons -j$(nproc)" - - name: Create Test Report - timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }} - run: > - ${{ env.RUN }} "PYTHONWARNINGS=ignore && - source selfdrive/test/setup_xvfb.sh && - CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py && - chmod -R 777 /tmp/comma_download_cache" - - name: Upload Test Report - uses: actions/upload-artifact@v4 - with: - name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} - path: selfdrive/ui/tests/test_ui/report_1/screenshots diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml deleted file mode 100644 index 98a3913600b9f8..00000000000000 --- a/.github/workflows/setup-with-retry/action.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: 'openpilot env setup, with retry on failure' - -inputs: - docker_hub_pat: - description: 'Auth token for Docker Hub, required for BuildJet jobs' - required: false - default: '' - sleep_time: - description: 'Time to sleep between retries' - required: false - default: 30 - -outputs: - duration: - description: 'Duration of the setup process in seconds' - value: ${{ steps.get_duration.outputs.duration }} - -runs: - using: "composite" - steps: - - id: start_time - shell: bash - run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV - - id: setup1 - uses: ./.github/workflows/setup - continue-on-error: true - with: - is_retried: true - - if: steps.setup1.outcome == 'failure' - shell: bash - run: sleep ${{ inputs.sleep_time }} - - id: setup2 - if: steps.setup1.outcome == 'failure' - uses: ./.github/workflows/setup - continue-on-error: true - with: - is_retried: true - - if: steps.setup2.outcome == 'failure' - shell: bash - run: sleep ${{ inputs.sleep_time }} - - id: setup3 - if: steps.setup2.outcome == 'failure' - uses: ./.github/workflows/setup - with: - is_retried: true - - id: get_duration - shell: bash - run: | - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME)) - echo "Total duration: $DURATION seconds" - echo "duration=$DURATION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml deleted file mode 100644 index 818060c3b010cc..00000000000000 --- a/.github/workflows/setup/action.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: 'openpilot env setup' - -inputs: - is_retried: - description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly' - required: false - default: 'false' - -runs: - using: "composite" - steps: - # assert that this action is retried using the setup-with-retry - - shell: bash - if: ${{ inputs.is_retried == 'false' }} - run: | - echo "You should not run this action directly. Use setup-with-retry instead" - exit 1 - - - shell: bash - name: No retries! - run: | - if [ "${{ github.run_attempt }}" -gt 1 ]; then - echo -e "\033[0;31m##################################################" - echo -e "\033[0;31m Retries not allowed! Fix the flaky test! " - echo -e "\033[0;31m##################################################\033[0m" - exit 1 - fi - - # do this after checkout to ensure our custom LFS config is used to pull from GitLab - - shell: bash - run: git lfs pull - - # build cache - - id: date - shell: bash - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV - - shell: bash - run: echo "$CACHE_COMMIT_DATE" - - id: scons-cache - uses: ./.github/workflows/auto-cache - with: - path: .ci_cache/scons_cache - key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} - restore-keys: | - scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }} - scons-${{ runner.arch }} - # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001 - - id: normalize-file-permissions - shell: bash - name: Normalize file permissions to ensure a consistent docker build cache - run: | - find . -type f -executable -not -perm 755 -exec chmod 755 {} \; - find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \; - # build our docker image - - shell: bash - run: eval ${{ env.BUILD }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml deleted file mode 100644 index 823df2b580e3ce..00000000000000 --- a/.github/workflows/stale.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: stale -on: - schedule: - - cron: '30 1 * * *' - workflow_dispatch: - -env: - DAYS_BEFORE_PR_CLOSE: 2 - DAYS_BEFORE_PR_STALE: 9 - DAYS_BEFORE_PR_STALE_DRAFT: 30 - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v9 - with: - exempt-all-milestones: true - - # pull request config - stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.' - close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.' - stale-pr-label: stale - delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo - exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale - days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }} - days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} - exempt-draft-pr: false - - # issue config - days-before-issue-stale: -1 # ignore issues for now - - # same as above, but give draft PRs more time - stale_drafts: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v9 - with: - exempt-all-milestones: true - - # pull request config - stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.' - close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.' - stale-pr-label: stale - delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo - exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale - days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} - days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} - exempt-draft-pr: true - - # issue config - days-before-issue-stale: -1 # ignore issues for now diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml deleted file mode 100644 index 9ec7a592230fb9..00000000000000 --- a/.github/workflows/ui_preview.yaml +++ /dev/null @@ -1,174 +0,0 @@ -name: "ui preview" -on: - push: - branches: - - master - pull_request_target: - types: [assigned, opened, synchronize, reopened, edited] - branches: - - 'master' - paths: - - 'selfdrive/ui/**' - workflow_dispatch: - -env: - UI_JOB_NAME: "Create UI Report" - REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} - SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }} - BRANCH_NAME: "openpilot/pr-${{ github.event.number }}" - -jobs: - preview: - #if: github.repository == 'commaai/openpilot' - if: false # FIXME: FrameReader is broken on CI runners - name: preview - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - pull-requests: write - actions: read - steps: - - name: Waiting for ui generation to start - run: sleep 30 - - - name: Waiting for ui generation to end - uses: lewagon/wait-on-check-action@v1.3.4 - with: - ref: ${{ env.SHA }} - check-name: ${{ env.UI_JOB_NAME }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - allowed-conclusions: success - wait-interval: 20 - - - name: Getting workflow run ID - id: get_run_id - run: | - echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?[0-9]+)") | .number')" >> $GITHUB_OUTPUT - - - name: Getting proposed ui - id: download-artifact - uses: dawidd6/action-download-artifact@v6 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ steps.get_run_id.outputs.run_id }} - search_artifacts: true - name: report-1-${{ env.REPORT_NAME }} - path: ${{ github.workspace }}/pr_ui - - - name: Getting master ui - uses: actions/checkout@v4 - with: - repository: commaai/ci-artifacts - ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} - path: ${{ github.workspace }}/master_ui - ref: openpilot_master_ui - - - name: Saving new master ui - if: github.ref == 'refs/heads/master' && github.event_name == 'push' - working-directory: ${{ github.workspace }}/master_ui - run: | - git checkout --orphan=new_master_ui - git rm -rf * - git branch -D openpilot_master_ui - git branch -m openpilot_master_ui - git config user.name "GitHub Actions Bot" - git config user.email "<>" - mv ${{ github.workspace }}/pr_ui/*.png . - git add . - git commit -m "screenshots for commit ${{ env.SHA }}" - git push origin openpilot_master_ui --force - - - name: Finding diff - if: github.event_name == 'pull_request_target' - id: find_diff - run: >- - sudo apt-get update && sudo apt-get install -y imagemagick - - scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device') - A=($scenes) - - DIFF="" - TABLE="
All Screenshots" - TABLE="${TABLE}" - - for ((i=0; i<${#A[*]}; i=i+1)); - do - # Check if the master file exists - if [ ! -f "${{ github.workspace }}/master_ui/${A[$i]}.png" ]; then - # This is a new file in PR UI that doesn't exist in master - DIFF="${DIFF}
" - DIFF="${DIFF}${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$" - DIFF="${DIFF}
" - - DIFF="${DIFF}" - DIFF="${DIFF} " - DIFF="${DIFF}" - - DIFF="${DIFF}
" - DIFF="${DIFF}
" - elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then - convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png - composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png - convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif - - mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png - - DIFF="${DIFF}
" - DIFF="${DIFF}${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$" - DIFF="${DIFF}" - - DIFF="${DIFF}" - DIFF="${DIFF} " - DIFF="${DIFF} " - DIFF="${DIFF}" - - DIFF="${DIFF}" - DIFF="${DIFF} " - DIFF="${DIFF} " - DIFF="${DIFF}" - - DIFF="${DIFF}
master proposed
diff composite diff
" - DIFF="${DIFF}
" - else - rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png - fi - - INDEX=$(($i % 2)) - if [[ $INDEX -eq 0 ]]; then - TABLE="${TABLE}" - fi - TABLE="${TABLE} " - if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then - TABLE="${TABLE}" - fi - done - - TABLE="${TABLE}" - - echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT" - - - name: Saving proposed ui - if: github.event_name == 'pull_request_target' - working-directory: ${{ github.workspace }}/master_ui - run: | - git config user.name "GitHub Actions Bot" - git config user.email "<>" - git checkout --orphan=${{ env.BRANCH_NAME }} - git rm -rf * - mv ${{ github.workspace }}/pr_ui/* . - git add . - git commit -m "screenshots for PR #${{ github.event.number }}" - git push origin ${{ env.BRANCH_NAME }} --force - - - name: Comment Screenshots on PR - if: github.event_name == 'pull_request_target' - uses: thollander/actions-comment-pull-request@v2 - with: - message: | - - ## UI Preview - ${{ steps.find_diff.outputs.DIFF }} - comment_tag: run_id_screenshots - pr_number: ${{ github.event.number }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0ce5a81c211402530206b2641c5b5eb3b8d517aa Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 19 Sep 2025 23:46:46 -0400 Subject: [PATCH 02/16] Revert "start over" This reverts commit 680f73c3a1110e49363237742145060a8018ab0d. --- .github/workflows/auto-cache/action.yaml | 58 ++++ .github/workflows/auto_pr_review.yaml | 52 ++++ .github/workflows/badges.yaml | 37 +++ .github/workflows/ci_weekly_report.yaml | 101 +++++++ .github/workflows/ci_weekly_run.yaml | 17 ++ .../workflows/compile-openpilot/action.yaml | 21 ++ .github/workflows/docs.yaml | 65 ++++ .github/workflows/jenkins-pr-trigger.yaml | 59 ++++ .github/workflows/model_review.yaml | 42 +++ .github/workflows/prebuilt.yaml | 39 +++ .github/workflows/release.yaml | 42 +++ .github/workflows/repo-maintenance.yaml | 72 +++++ .github/workflows/selfdrive_tests.yaml | 279 ++++++++++++++++++ .../workflows/setup-with-retry/action.yaml | 52 ++++ .github/workflows/setup/action.yaml | 56 ++++ .github/workflows/stale.yaml | 52 ++++ .github/workflows/ui_preview.yaml | 174 +++++++++++ 17 files changed, 1218 insertions(+) create mode 100644 .github/workflows/auto-cache/action.yaml create mode 100644 .github/workflows/auto_pr_review.yaml create mode 100644 .github/workflows/badges.yaml create mode 100644 .github/workflows/ci_weekly_report.yaml create mode 100644 .github/workflows/ci_weekly_run.yaml create mode 100644 .github/workflows/compile-openpilot/action.yaml create mode 100644 .github/workflows/docs.yaml create mode 100644 .github/workflows/jenkins-pr-trigger.yaml create mode 100644 .github/workflows/model_review.yaml create mode 100644 .github/workflows/prebuilt.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/repo-maintenance.yaml create mode 100644 .github/workflows/selfdrive_tests.yaml create mode 100644 .github/workflows/setup-with-retry/action.yaml create mode 100644 .github/workflows/setup/action.yaml create mode 100644 .github/workflows/stale.yaml create mode 100644 .github/workflows/ui_preview.yaml diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml new file mode 100644 index 00000000000000..377b1eedcde4bb --- /dev/null +++ b/.github/workflows/auto-cache/action.yaml @@ -0,0 +1,58 @@ +name: 'automatically cache based on current runner' + +inputs: + path: + description: 'path to cache' + required: true + key: + description: 'key' + required: true + restore-keys: + description: 'restore-keys' + required: true + save: + description: 'whether to save the cache' + default: 'true' + required: false +outputs: + cache-hit: + description: 'cache hit occurred' + value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) || + (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) || + (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }} + +runs: + using: "composite" + steps: + - name: setup namespace cache + id: ns-cache + if: ${{ contains(runner.name, 'nsc') }} + uses: namespacelabs/nscloud-cache-action@v1 + with: + path: ${{ inputs.path }} + + - name: setup github cache + id: gha-cache + if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }} + uses: 'actions/cache@v4' + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} + + - name: setup github cache + id: gha-cache-ro + if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }} + uses: 'actions/cache/restore@v4' + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} + + # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps + - id: scons-cache-setup + shell: bash + run: | + mkdir -p ${{ inputs.path }} + sudo chmod -R 777 ${{ inputs.path }} + sudo chown -R $USER ${{ inputs.path }} diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml new file mode 100644 index 00000000000000..c6a1cb98219a68 --- /dev/null +++ b/.github/workflows/auto_pr_review.yaml @@ -0,0 +1,52 @@ +name: "PR review" +on: + pull_request_target: + types: [opened, reopened, synchronize, edited] + +jobs: + labeler: + name: review + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: false + + # Label PRs + - uses: actions/labeler@v5.0.0 + with: + dot: true + configuration-path: .github/labeler.yaml + + # Check PR target branch + - name: check branch + uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5 + if: github.repository == 'commaai/openpilot' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + target: /^(?!master$).*/ + exclude: /commaai:.*/ + change-to: ${{ github.base_ref }} + already-exists-action: close_this + already-exists-comment: "Your PR should be made against the `master` branch" + + # Welcome comment + - name: "First timers PR" + uses: actions/first-interaction@v1 + if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + pr-message: | + + Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following: + * Convert your PR to a draft unless it's ready to review + * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md) + * Before marking as "ready for review", ensure: + * the goal is clearly stated in the description + * all the tests are passing + * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged) + * include a route or your device' dongle ID if relevant diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml new file mode 100644 index 00000000000000..63ee736dcab84b --- /dev/null +++ b/.github/workflows/badges.yaml @@ -0,0 +1,37 @@ +name: badges +on: + schedule: + - cron: '0 * * * *' + workflow_dispatch: + +env: + BASE_IMAGE: openpilot-base + DOCKER_REGISTRY: ghcr.io/commaai + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c + +jobs: + badges: + name: create badges + runs-on: ubuntu-latest + if: github.repository == 'commaai/openpilot' + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Push badges + run: | + ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py" + + rm .gitattributes + + git checkout --orphan badges + git rm -rf --cached . + git config user.email "badge-researcher@comma.ai" + git config user.name "Badge Researcher" + + git add translation_badge.svg + git commit -m "Add/Update badges" + git push -f origin HEAD diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml new file mode 100644 index 00000000000000..9821283cb57442 --- /dev/null +++ b/.github/workflows/ci_weekly_report.yaml @@ -0,0 +1,101 @@ +name: weekly CI test report +on: + schedule: + - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday + workflow_dispatch: + inputs: + ci_runs: + description: 'The amount of runs to trigger in CI test report' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }} + +jobs: + setup: + if: github.repository == 'commaai/openpilot' + runs-on: ubuntu-latest + outputs: + ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }} + steps: + - id: ci_runs_setup + name: CI_RUNS=${{ env.CI_RUNS }} + run: | + matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))") + echo "matrix=$matrix" >> $GITHUB_OUTPUT + + ci_matrix_run: + needs: [ setup ] + strategy: + fail-fast: false + matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}} + uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master + with: + run_number: ${{ matrix.run_number }} + + report: + needs: [ci_matrix_run] + runs-on: ubuntu-latest + if: always() + steps: + - name: Get job results + uses: actions/github-script@v7 + id: get-job-results + with: + script: | + const jobs = await github + .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", { + owner: "commaai", + repo: "${{ github.event.repository.name }}", + run_id: "${{ github.run_id }}", + attempt: "${{ github.run_attempt }}", + }) + var report = {} + jobs.slice(1, jobs.length-1).forEach(job => { + if (job.conclusion === "skipped") return; + const jobName = job.name.split(" / ")[2]; + const runRegex = /\((.*?)\)/; + const run = job.name.match(runRegex)[1]; + report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] }; + switch (job.conclusion) { + case "success": + report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break; + case "failure": + report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break; + case "canceled": + report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break; + } + }); + return JSON.stringify({"jobs": report}); + + - name: Add job results to summary + env: + JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }} + run: | + cat <> template.html + + + + + + + + + + + {% for key in jobs.keys() %} + + + + + + {% endfor %} +
Job✅ Passing❌ Failure Details
{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}{{ key }}{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%{% if jobs[key]["failures"]|length > 0 %}
{% for failure in jobs[key]["failures"] %}Log for run #{{ failure['run_number'] }}
{% endfor %}
{% else %}{% endif %}
+ EOF + + pip install jinja2-cli + echo $JOB_RESULTS | jinja2 template.html > report.html + echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY + cat report.html >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/ci_weekly_run.yaml b/.github/workflows/ci_weekly_run.yaml new file mode 100644 index 00000000000000..d8bf91116d794f --- /dev/null +++ b/.github/workflows/ci_weekly_run.yaml @@ -0,0 +1,17 @@ +name: weekly CI test run +on: + workflow_call: + inputs: + run_number: + required: true + type: string + +concurrency: + group: ci-run-${{ inputs.run_number }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + selfdrive_tests: + uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master + with: + run_number: ${{ inputs.run_number }} diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml new file mode 100644 index 00000000000000..4015746c0e3680 --- /dev/null +++ b/.github/workflows/compile-openpilot/action.yaml @@ -0,0 +1,21 @@ +name: 'compile openpilot' + +runs: + using: "composite" + steps: + - shell: bash + name: Build openpilot with all flags + run: | + ${{ env.RUN }} "scons -j$(nproc)" + ${{ env.RUN }} "release/check-dirty.sh" + - shell: bash + name: Cleanup scons cache and rebuild + run: | + ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ + scons -j$(nproc) --cache-populate" + - name: Save scons cache + uses: actions/cache/save@v4 + if: github.ref == 'refs/heads/master' + with: + path: .ci_cache/scons_cache + key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000000000..92c311829c0654 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,65 @@ +name: docs + +on: + push: + branches: + - master + pull_request: + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string +concurrency: + group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + docs: + name: build docs + runs-on: ubuntu-24.04 + steps: + - uses: commaai/timeout@v1 + + - uses: actions/checkout@v4 + with: + submodules: true + + # Build + - name: Build docs + run: | + # TODO: can we install just the "docs" dependency group without the normal deps? + pip install mkdocs + mkdocs build + + # Push to docs.comma.ai + - uses: actions/checkout@v4 + if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' + with: + path: openpilot-docs + ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }} + repository: commaai/openpilot-docs + - name: Push + if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' + run: | + set -x + + source release/identity.sh + + cd openpilot-docs + git checkout --orphan tmp + git rm -rf . + + # copy over docs + cp -r ../docs_site/ docs/ + + # GitHub pages config + touch docs/.nojekyll + echo -n docs.comma.ai > docs/CNAME + + git add -f . + git commit -m "build docs" + + # docs live in different repo to not bloat openpilot's full clone size + git push -f origin tmp:gh-pages diff --git a/.github/workflows/jenkins-pr-trigger.yaml b/.github/workflows/jenkins-pr-trigger.yaml new file mode 100644 index 00000000000000..14e2fdf49ba970 --- /dev/null +++ b/.github/workflows/jenkins-pr-trigger.yaml @@ -0,0 +1,59 @@ +name: jenkins scan + +on: + issue_comment: + types: [created, edited] + +jobs: + # TODO: gc old branches in a separate job in this workflow + scan-comments: + runs-on: ubuntu-latest + if: ${{ github.event.issue.pull_request }} + permissions: + contents: write + issues: write + steps: + - name: Check for trigger phrase + id: check_comment + uses: actions/github-script@v7 + with: + script: | + const triggerPhrase = "trigger-jenkins"; + const comment = context.payload.comment.body; + const commenter = context.payload.comment.user.login; + + const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: commenter + }); + + const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin'; + + return (hasWriteAccess && comment.includes(triggerPhrase)); + result-encoding: json + + - name: Checkout repository + if: steps.check_comment.outputs.result == 'true' + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + + - name: Push to tmp-jenkins branch + if: steps.check_comment.outputs.result == 'true' + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b tmp-jenkins-${{ github.event.issue.number }} + GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }} + + - name: Delete trigger comment + if: steps.check_comment.outputs.result == 'true' && always() + uses: actions/github-script@v7 + with: + script: | + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + }); diff --git a/.github/workflows/model_review.yaml b/.github/workflows/model_review.yaml new file mode 100644 index 00000000000000..0e1825864c278c --- /dev/null +++ b/.github/workflows/model_review.yaml @@ -0,0 +1,42 @@ +name: "model review" + +on: + pull_request: + types: [opened, reopened, synchronize] + paths: + - 'selfdrive/modeld/models/*.onnx' + workflow_dispatch: + +jobs: + comment: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + if: github.repository == 'commaai/openpilot' + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Checkout master + uses: actions/checkout@v4 + with: + ref: master + path: base + - run: git lfs pull + - run: cd base && git lfs pull + + - run: pip install onnx + + - name: scripts/reporter.py + id: report + run: | + echo "content<> $GITHUB_OUTPUT + echo "## Model Review" >> $GITHUB_OUTPUT + MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Post model report comment + uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16 + with: + header: model-review + message: ${{ steps.report.outputs.content }} \ No newline at end of file diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml new file mode 100644 index 00000000000000..d8963ec89f4f93 --- /dev/null +++ b/.github/workflows/prebuilt.yaml @@ -0,0 +1,39 @@ +name: prebuilt +on: + schedule: + - cron: '0 * * * *' + workflow_dispatch: + +env: + DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + BUILD: selfdrive/test/docker_build.sh prebuilt + +jobs: + build_prebuilt: + name: build prebuilt + runs-on: ubuntu-latest + if: github.repository == 'commaai/openpilot' + env: + PUSH_IMAGE: true + permissions: + checks: read + contents: read + packages: write + steps: + - name: Wait for green check mark + if: ${{ github.event_name != 'workflow_dispatch' }} + uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc + with: + ref: master + wait-interval: 30 + running-workflow-name: 'build prebuilt' + repo-token: ${{ secrets.GITHUB_TOKEN }} + check-regexp: ^((?!.*(build master-ci).*).)*$ + - uses: actions/checkout@v4 + with: + submodules: true + - run: git lfs pull + - name: Build and Push docker image + run: | + $DOCKER_LOGIN + eval "$BUILD" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000000000..0f4ce6cb3a718e --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,42 @@ +name: release +on: + schedule: + - cron: '0 9 * * *' + workflow_dispatch: + +jobs: + build_masterci: + name: build master-ci + env: + ImageOS: ubuntu24 + container: + image: ghcr.io/commaai/openpilot-base:latest + runs-on: ubuntu-latest + if: github.repository == 'commaai/openpilot' + permissions: + checks: read + contents: write + steps: + - name: Install wait-on-check-action dependencies + run: | + sudo apt-get update + sudo apt-get install -y libyaml-dev + - name: Wait for green check mark + if: ${{ github.event_name == 'schedule' }} + uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc + with: + ref: master + wait-interval: 30 + running-workflow-name: 'build master-ci' + repo-token: ${{ secrets.GITHUB_TOKEN }} + check-regexp: ^((?!.*(build prebuilt).*).)*$ + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Pull LFS + run: | + git config --global --add safe.directory '*' + git lfs pull + - name: Push master-ci + run: BRANCH=__nightly release/build_stripped.sh diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml new file mode 100644 index 00000000000000..7bb91c0ca4fe04 --- /dev/null +++ b/.github/workflows/repo-maintenance.yaml @@ -0,0 +1,72 @@ +name: repo maintenance + +on: + schedule: + - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST) + workflow_dispatch: + +env: + BASE_IMAGE: openpilot-base + BUILD: selfdrive/test/docker_build.sh base + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + +jobs: + update_translations: + runs-on: ubuntu-latest + if: github.repository == 'commaai/openpilot' + steps: + - uses: actions/checkout@v4 + - uses: ./.github/workflows/setup-with-retry + - name: Update translations + run: | + ${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish" + - name: Create Pull Request + uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 + with: + author: Vehicle Researcher + commit-message: "Update translations" + title: "[bot] Update translations" + body: "Automatic PR from repo-maintenance -> update_translations" + branch: "update-translations" + base: "master" + delete-branch: true + labels: bot + + package_updates: + name: package_updates + runs-on: ubuntu-latest + container: + image: ghcr.io/commaai/openpilot-base:latest + if: github.repository == 'commaai/openpilot' + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: uv lock + run: | + python3 -m ensurepip --upgrade + pip3 install uv + uv lock --upgrade + - name: bump submodules + run: | + git config --global --add safe.directory '*' + git submodule update --remote + git add . + - name: update car docs + run: | + export PYTHONPATH="$PWD" + scons -j$(nproc) --minimal opendbc_repo + python selfdrive/car/docs.py + git add docs/CARS.md + - name: Create Pull Request + uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 + with: + author: Vehicle Researcher + token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} + commit-message: Update Python packages + title: '[bot] Update Python packages' + branch: auto-package-updates + base: master + delete-branch: true + body: 'Automatic PR from repo-maintenance -> package_updates' + labels: bot diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml new file mode 100644 index 00000000000000..beb426c669719b --- /dev/null +++ b/.github/workflows/selfdrive_tests.yaml @@ -0,0 +1,279 @@ +name: selfdrive + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string + +concurrency: + group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + cancel-in-progress: true + +env: + PYTHONWARNINGS: error + BASE_IMAGE: openpilot-base + AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} + + DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + BUILD: selfdrive/test/docker_build.sh base + + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + + PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical + +jobs: + build_release: + name: build release + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + env: + STRIPPED_DIR: /tmp/releasepilot + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Getting LFS files + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e + with: + timeout_minutes: 2 + max_attempts: 3 + command: git lfs pull + - name: Build devel + timeout-minutes: 1 + run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh + - uses: ./.github/workflows/setup-with-retry + - name: Build openpilot and run checks + timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache + run: | + cd $STRIPPED_DIR + ${{ env.RUN }} "python3 system/manager/build.py" + - name: Run tests + timeout-minutes: 1 + run: | + cd $STRIPPED_DIR + ${{ env.RUN }} "release/check-dirty.sh" + - name: Check submodules + if: github.repository == 'commaai/openpilot' + timeout-minutes: 3 + run: release/check-submodules.sh + + build: + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup docker push + if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' + run: | + echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" + $DOCKER_LOGIN + - uses: ./.github/workflows/setup-with-retry + - uses: ./.github/workflows/compile-openpilot + timeout-minutes: 30 + + build_mac: + name: build macOS + runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV + - name: Homebrew cache + uses: ./.github/workflows/auto-cache + with: + path: ~/Library/Caches/Homebrew + key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + restore-keys: | + brew-macos-${{ env.CACHE_COMMIT_DATE }} + brew-macos + - name: Install dependencies + run: ./tools/mac_setup.sh + env: + PYTHONWARNINGS: default # package install has DeprecationWarnings + HOMEBREW_DISPLAY_INSTALL_TIMES: 1 + - run: git lfs pull + - name: Getting scons cache + uses: ./.github/workflows/auto-cache + with: + path: /tmp/scons_cache + key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + restore-keys: | + scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }} + scons-${{ runner.arch }}-macos + - name: Building openpilot + run: . .venv/bin/activate && scons -j$(nproc) + + static_analysis: + name: static analysis + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + env: + PYTHONWARNINGS: default + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Static analysis + timeout-minutes: 1 + run: ${{ env.RUN }} "scripts/lint/lint.sh" + + unit_tests: + name: unit tests + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + id: setup-step + - name: Build openpilot + run: ${{ env.RUN }} "scons -j$(nproc)" + - name: Run unit tests + timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} + run: | + ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ + # Pre-compile Python bytecode so each pytest worker doesn't need to + $PYTEST --collect-only -m 'not slow' -qq && \ + MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \ + ./selfdrive/ui/tests/create_test_translations.sh && \ + QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ + chmod -R 777 /tmp/comma_download_cache" + + process_replay: + name: process replay + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + id: setup-step + - name: Cache test routes + id: dependency-cache + uses: actions/cache@v4 + with: + path: .ci_cache/comma_download_cache + key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }} + - name: Build openpilot + run: | + ${{ env.RUN }} "scons -j$(nproc)" + - name: Run replay + timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }} + run: | + ${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ + chmod -R 777 /tmp/comma_download_cache" + - name: Print diff + id: print-diff + if: always() + run: cat selfdrive/test/process_replay/diff.txt + - uses: actions/upload-artifact@v4 + if: always() + continue-on-error: true + with: + name: process_replay_diff.txt + path: selfdrive/test/process_replay/diff.txt + - name: Upload reference logs + if: false # TODO: move this to github instead of azure + run: | + ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only" + - name: Run regen + if: false + timeout-minutes: 4 + run: | + ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ + chmod -R 777 /tmp/comma_download_cache" + + simulator_driving: + name: simulator driving + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + id: setup-step + - name: Build openpilot + run: | + ${{ env.RUN }} "scons -j$(nproc)" + - name: Driving test + timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }} + run: | + ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ + source selfdrive/test/setup_vsound.sh && \ + CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py" + + create_ui_report: + # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml + name: Create UI Report + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + if: false # FIXME: FrameReader is broken on CI runners + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: caching frames + id: frames-cache + uses: actions/cache@v4 + with: + path: .ci_cache/comma_download_cache + key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }} + - name: Build openpilot + run: ${{ env.RUN }} "scons -j$(nproc)" + - name: Create Test Report + timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }} + run: > + ${{ env.RUN }} "PYTHONWARNINGS=ignore && + source selfdrive/test/setup_xvfb.sh && + CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py && + chmod -R 777 /tmp/comma_download_cache" + - name: Upload Test Report + uses: actions/upload-artifact@v4 + with: + name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} + path: selfdrive/ui/tests/test_ui/report_1/screenshots diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml new file mode 100644 index 00000000000000..98a3913600b9f8 --- /dev/null +++ b/.github/workflows/setup-with-retry/action.yaml @@ -0,0 +1,52 @@ +name: 'openpilot env setup, with retry on failure' + +inputs: + docker_hub_pat: + description: 'Auth token for Docker Hub, required for BuildJet jobs' + required: false + default: '' + sleep_time: + description: 'Time to sleep between retries' + required: false + default: 30 + +outputs: + duration: + description: 'Duration of the setup process in seconds' + value: ${{ steps.get_duration.outputs.duration }} + +runs: + using: "composite" + steps: + - id: start_time + shell: bash + run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV + - id: setup1 + uses: ./.github/workflows/setup + continue-on-error: true + with: + is_retried: true + - if: steps.setup1.outcome == 'failure' + shell: bash + run: sleep ${{ inputs.sleep_time }} + - id: setup2 + if: steps.setup1.outcome == 'failure' + uses: ./.github/workflows/setup + continue-on-error: true + with: + is_retried: true + - if: steps.setup2.outcome == 'failure' + shell: bash + run: sleep ${{ inputs.sleep_time }} + - id: setup3 + if: steps.setup2.outcome == 'failure' + uses: ./.github/workflows/setup + with: + is_retried: true + - id: get_duration + shell: bash + run: | + END_TIME=$(date +%s) + DURATION=$((END_TIME - START_TIME)) + echo "Total duration: $DURATION seconds" + echo "duration=$DURATION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml new file mode 100644 index 00000000000000..818060c3b010cc --- /dev/null +++ b/.github/workflows/setup/action.yaml @@ -0,0 +1,56 @@ +name: 'openpilot env setup' + +inputs: + is_retried: + description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly' + required: false + default: 'false' + +runs: + using: "composite" + steps: + # assert that this action is retried using the setup-with-retry + - shell: bash + if: ${{ inputs.is_retried == 'false' }} + run: | + echo "You should not run this action directly. Use setup-with-retry instead" + exit 1 + + - shell: bash + name: No retries! + run: | + if [ "${{ github.run_attempt }}" -gt 1 ]; then + echo -e "\033[0;31m##################################################" + echo -e "\033[0;31m Retries not allowed! Fix the flaky test! " + echo -e "\033[0;31m##################################################\033[0m" + exit 1 + fi + + # do this after checkout to ensure our custom LFS config is used to pull from GitLab + - shell: bash + run: git lfs pull + + # build cache + - id: date + shell: bash + run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV + - shell: bash + run: echo "$CACHE_COMMIT_DATE" + - id: scons-cache + uses: ./.github/workflows/auto-cache + with: + path: .ci_cache/scons_cache + key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + restore-keys: | + scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }} + scons-${{ runner.arch }} + # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001 + - id: normalize-file-permissions + shell: bash + name: Normalize file permissions to ensure a consistent docker build cache + run: | + find . -type f -executable -not -perm 755 -exec chmod 755 {} \; + find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \; + # build our docker image + - shell: bash + run: eval ${{ env.BUILD }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 00000000000000..823df2b580e3ce --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,52 @@ +name: stale +on: + schedule: + - cron: '30 1 * * *' + workflow_dispatch: + +env: + DAYS_BEFORE_PR_CLOSE: 2 + DAYS_BEFORE_PR_STALE: 9 + DAYS_BEFORE_PR_STALE_DRAFT: 30 + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + exempt-all-milestones: true + + # pull request config + stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.' + close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.' + stale-pr-label: stale + delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo + exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale + days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }} + days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} + exempt-draft-pr: false + + # issue config + days-before-issue-stale: -1 # ignore issues for now + + # same as above, but give draft PRs more time + stale_drafts: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + exempt-all-milestones: true + + # pull request config + stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.' + close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.' + stale-pr-label: stale + delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo + exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale + days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} + days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} + exempt-draft-pr: true + + # issue config + days-before-issue-stale: -1 # ignore issues for now diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml new file mode 100644 index 00000000000000..9ec7a592230fb9 --- /dev/null +++ b/.github/workflows/ui_preview.yaml @@ -0,0 +1,174 @@ +name: "ui preview" +on: + push: + branches: + - master + pull_request_target: + types: [assigned, opened, synchronize, reopened, edited] + branches: + - 'master' + paths: + - 'selfdrive/ui/**' + workflow_dispatch: + +env: + UI_JOB_NAME: "Create UI Report" + REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} + SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }} + BRANCH_NAME: "openpilot/pr-${{ github.event.number }}" + +jobs: + preview: + #if: github.repository == 'commaai/openpilot' + if: false # FIXME: FrameReader is broken on CI runners + name: preview + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + actions: read + steps: + - name: Waiting for ui generation to start + run: sleep 30 + + - name: Waiting for ui generation to end + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ env.SHA }} + check-name: ${{ env.UI_JOB_NAME }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + allowed-conclusions: success + wait-interval: 20 + + - name: Getting workflow run ID + id: get_run_id + run: | + echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?[0-9]+)") | .number')" >> $GITHUB_OUTPUT + + - name: Getting proposed ui + id: download-artifact + uses: dawidd6/action-download-artifact@v6 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ steps.get_run_id.outputs.run_id }} + search_artifacts: true + name: report-1-${{ env.REPORT_NAME }} + path: ${{ github.workspace }}/pr_ui + + - name: Getting master ui + uses: actions/checkout@v4 + with: + repository: commaai/ci-artifacts + ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} + path: ${{ github.workspace }}/master_ui + ref: openpilot_master_ui + + - name: Saving new master ui + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + working-directory: ${{ github.workspace }}/master_ui + run: | + git checkout --orphan=new_master_ui + git rm -rf * + git branch -D openpilot_master_ui + git branch -m openpilot_master_ui + git config user.name "GitHub Actions Bot" + git config user.email "<>" + mv ${{ github.workspace }}/pr_ui/*.png . + git add . + git commit -m "screenshots for commit ${{ env.SHA }}" + git push origin openpilot_master_ui --force + + - name: Finding diff + if: github.event_name == 'pull_request_target' + id: find_diff + run: >- + sudo apt-get update && sudo apt-get install -y imagemagick + + scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device') + A=($scenes) + + DIFF="" + TABLE="
All Screenshots" + TABLE="${TABLE}" + + for ((i=0; i<${#A[*]}; i=i+1)); + do + # Check if the master file exists + if [ ! -f "${{ github.workspace }}/master_ui/${A[$i]}.png" ]; then + # This is a new file in PR UI that doesn't exist in master + DIFF="${DIFF}
" + DIFF="${DIFF}${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$" + DIFF="${DIFF}
" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}
" + DIFF="${DIFF}
" + elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then + convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png + composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png + convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif + + mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png + + DIFF="${DIFF}
" + DIFF="${DIFF}${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$" + DIFF="${DIFF}" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}
master proposed
diff composite diff
" + DIFF="${DIFF}
" + else + rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png + fi + + INDEX=$(($i % 2)) + if [[ $INDEX -eq 0 ]]; then + TABLE="${TABLE}" + fi + TABLE="${TABLE} " + if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then + TABLE="${TABLE}" + fi + done + + TABLE="${TABLE}" + + echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT" + + - name: Saving proposed ui + if: github.event_name == 'pull_request_target' + working-directory: ${{ github.workspace }}/master_ui + run: | + git config user.name "GitHub Actions Bot" + git config user.email "<>" + git checkout --orphan=${{ env.BRANCH_NAME }} + git rm -rf * + mv ${{ github.workspace }}/pr_ui/* . + git add . + git commit -m "screenshots for PR #${{ github.event.number }}" + git push origin ${{ env.BRANCH_NAME }} --force + + - name: Comment Screenshots on PR + if: github.event_name == 'pull_request_target' + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + + ## UI Preview + ${{ steps.find_diff.outputs.DIFF }} + comment_tag: run_id_screenshots + pr_number: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6791dba96387f481c354a48448562ae93009918f Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 19 Sep 2025 23:51:28 -0400 Subject: [PATCH 03/16] try --- Dockerfile.openpilot_base | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 44d8d95e95d926..916df3a794f769 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -62,14 +62,8 @@ ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id ARG USER=batman -ARG USER_UID=1001 -RUN useradd -m -s /bin/bash -u $USER_UID $USER -RUN usermod -aG sudo $USER -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -USER $USER - -COPY --chown=$USER pyproject.toml uv.lock /home/$USER -COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/ +COPY pyproject.toml uv.lock /home/$USER +COPY tools/install_python_dependencies.sh /home/$USER/tools/ ENV VIRTUAL_ENV=/home/$USER/.venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" @@ -77,5 +71,4 @@ RUN cd /home/$USER && \ tools/install_python_dependencies.sh && \ rm -rf tools/ pyproject.toml uv.lock .cache -USER root RUN sudo git config --global --add safe.directory /tmp/openpilot From a9143a7e227f6039a12879e68d910101c99b45ad Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 19 Sep 2025 23:56:59 -0400 Subject: [PATCH 04/16] try more --- Dockerfile.openpilot_base | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 916df3a794f769..d706e69047ef9a 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -62,13 +62,12 @@ ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id ARG USER=batman -COPY pyproject.toml uv.lock /home/$USER -COPY tools/install_python_dependencies.sh /home/$USER/tools/ +COPY tools/install_python_dependencies.sh pyproject.toml uv.lock /home/$USER ENV VIRTUAL_ENV=/home/$USER/.venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN cd /home/$USER && \ - tools/install_python_dependencies.sh && \ - rm -rf tools/ pyproject.toml uv.lock .cache + install_python_dependencies.sh && \ + rm -rf install_python_dependencies.sh pyproject.toml uv.lock .cache RUN sudo git config --global --add safe.directory /tmp/openpilot From 89f50e846b69c1cf37481e52f1c2982af467a955 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 19 Sep 2025 23:59:59 -0400 Subject: [PATCH 05/16] more --- Dockerfile.openpilot_base | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index d706e69047ef9a..ca72fc0823d480 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -62,12 +62,12 @@ ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id ARG USER=batman -COPY tools/install_python_dependencies.sh pyproject.toml uv.lock /home/$USER +WORKDIR /home/$USER +COPY tools/install_python_dependencies.sh pyproject.toml uv.lock . ENV VIRTUAL_ENV=/home/$USER/.venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN cd /home/$USER && \ - install_python_dependencies.sh && \ +RUN install_python_dependencies.sh && \ rm -rf install_python_dependencies.sh pyproject.toml uv.lock .cache RUN sudo git config --global --add safe.directory /tmp/openpilot From af9da0f764cc32454827ab59f4fea1dd3559f237 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:02:10 -0400 Subject: [PATCH 06/16] more --- Dockerfile.openpilot_base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index ca72fc0823d480..75cc030b1de8a1 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -67,7 +67,7 @@ COPY tools/install_python_dependencies.sh pyproject.toml uv.lock . ENV VIRTUAL_ENV=/home/$USER/.venv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN install_python_dependencies.sh && \ +RUN ./install_python_dependencies.sh && \ rm -rf install_python_dependencies.sh pyproject.toml uv.lock .cache RUN sudo git config --global --add safe.directory /tmp/openpilot From 8caaf11fb9af6472f3fc088eec881983c8ecf37c Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:17:18 -0400 Subject: [PATCH 07/16] def legit copypasta --- Dockerfile.openpilot_base | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 75cc030b1de8a1..e244137121dd96 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -61,13 +61,19 @@ ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id -ARG USER=batman -WORKDIR /home/$USER -COPY tools/install_python_dependencies.sh pyproject.toml uv.lock . +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +ENV UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + UV_PYTHON_DOWNLOADS=never \ + UV_PYTHON=python3.12 \ + UV_PROJECT_ENVIRONMENT=/app -ENV VIRTUAL_ENV=/home/$USER/.venv -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN ./install_python_dependencies.sh && \ - rm -rf install_python_dependencies.sh pyproject.toml uv.lock .cache +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + uv sync \ + --locked \ + --no-dev \ + --no-install-project RUN sudo git config --global --add safe.directory /tmp/openpilot From 6fcdec8d1ded1de2022a975f2ddade95d3dcffa2 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:19:13 -0400 Subject: [PATCH 08/16] a bit more --- Dockerfile.openpilot_base | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index e244137121dd96..97e27ff61a1704 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -66,14 +66,12 @@ ENV UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_PYTHON_DOWNLOADS=never \ UV_PYTHON=python3.12 \ - UV_PROJECT_ENVIRONMENT=/app + UV_PROJECT_ENVIRONMENT=/tmp/openpilot RUN --mount=type=cache,target=/root/.cache \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ uv sync \ - --locked \ - --no-dev \ --no-install-project RUN sudo git config --global --add safe.directory /tmp/openpilot From 9b4f383e400732ebe5ecc1d64e7dd53c7791fe11 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:23:24 -0400 Subject: [PATCH 09/16] lil more --- Dockerfile.openpilot_base | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 97e27ff61a1704..ddb0541ace23f8 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -68,6 +68,8 @@ ENV UV_LINK_MODE=copy \ UV_PYTHON=python3.12 \ UV_PROJECT_ENVIRONMENT=/tmp/openpilot +WORKDIR /tmp/openpilot + RUN --mount=type=cache,target=/root/.cache \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ From abbf183c50148b8db8a06c89a15459c97067d80a Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:27:01 -0400 Subject: [PATCH 10/16] more --- Dockerfile.openpilot_base | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index ddb0541ace23f8..bd073202300a9e 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -63,10 +63,7 @@ RUN dbus-uuidgen > /etc/machine-id COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv ENV UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PYTHON_DOWNLOADS=never \ - UV_PYTHON=python3.12 \ - UV_PROJECT_ENVIRONMENT=/tmp/openpilot + UV_COMPILE_BYTECODE=1 WORKDIR /tmp/openpilot From 633153f6701012896e02c8c7d77a5ac91be47f70 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:29:47 -0400 Subject: [PATCH 11/16] more --- Dockerfile.openpilot_base | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index bd073202300a9e..49419f3739c53a 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -65,12 +65,10 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv ENV UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 -WORKDIR /tmp/openpilot +WORKDIR /home/batman/openpilot RUN --mount=type=cache,target=/root/.cache \ --mount=type=bind,source=uv.lock,target=uv.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ uv sync \ --no-install-project - -RUN sudo git config --global --add safe.directory /tmp/openpilot From 01a0f38592fc5bf81f288f69c853115370cc4f26 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:33:35 -0400 Subject: [PATCH 12/16] revert --- Dockerfile.openpilot_base | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 49419f3739c53a..44d8d95e95d926 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -61,14 +61,21 @@ ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv -ENV UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 +ARG USER=batman +ARG USER_UID=1001 +RUN useradd -m -s /bin/bash -u $USER_UID $USER +RUN usermod -aG sudo $USER +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER $USER -WORKDIR /home/batman/openpilot +COPY --chown=$USER pyproject.toml uv.lock /home/$USER +COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/ -RUN --mount=type=cache,target=/root/.cache \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync \ - --no-install-project +ENV VIRTUAL_ENV=/home/$USER/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN cd /home/$USER && \ + tools/install_python_dependencies.sh && \ + rm -rf tools/ pyproject.toml uv.lock .cache + +USER root +RUN sudo git config --global --add safe.directory /tmp/openpilot From 94d7a1813b874f0e2c8a9fe845dfed8b9eff7783 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:34:20 -0400 Subject: [PATCH 13/16] a different angle --- Dockerfile.openpilot_base | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 44d8d95e95d926..851530596e85ad 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -1,4 +1,4 @@ -FROM ubuntu:24.04 +FROM ubuntu:24.04 as builder ENV PYTHONUNBUFFERED=1 @@ -61,6 +61,7 @@ ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id +FROM builder as openpilot_base ARG USER=batman ARG USER_UID=1001 RUN useradd -m -s /bin/bash -u $USER_UID $USER From 1c8199c226b6727450c2f90b882c78b670867c31 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:40:21 -0400 Subject: [PATCH 14/16] retry with null change From 0e4bd99cacd9711b37952734aebdf38183846eea Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:51:54 -0400 Subject: [PATCH 15/16] don't think we need this --- Dockerfile.openpilot_base | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 851530596e85ad..2b46a5c909f54f 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -65,8 +65,6 @@ FROM builder as openpilot_base ARG USER=batman ARG USER_UID=1001 RUN useradd -m -s /bin/bash -u $USER_UID $USER -RUN usermod -aG sudo $USER -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER $USER COPY --chown=$USER pyproject.toml uv.lock /home/$USER From 1fe9a007da4698a9ffbfb86079520a2a2bb4eb18 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Sat, 20 Sep 2025 00:57:03 -0400 Subject: [PATCH 16/16] retry with null change