diff --git a/.helper_bash_functions b/.helper_bash_functions index d8a1a76..6f6bf8f 100644 --- a/.helper_bash_functions +++ b/.helper_bash_functions @@ -26,13 +26,33 @@ colcon_build() { START_DIR=$(pwd) && \ } colcon_build_this() { colcon_build "$(find_cmake_project_names_from_dir .)"; } rosdep_install() { rosdep install --from-paths src --ignore-src -r -y ; } +_show_test_failures() { + python3 - "$@" <<'PYEOF' +import sys, xml.etree.ElementTree as ET +from pathlib import Path +for d in sys.argv[1:]: + for p in sorted(Path(d).rglob("*.xml")): + for tc in ET.parse(p).iter("testcase"): + for f in list(tc.iter("failure")) + list(tc.iter("error")): + tag = "FAIL" if f.tag == "failure" else "ERROR" + print(f"\n {tag}: {tc.get('classname', '')}.{tc.get('name', '')}") + if f.text: + lines = f.text.strip().splitlines() + for l in lines[:20]: + print(f" {l}") + if len(lines) > 20: + print(f" ... ({len(lines) - 20} more lines)") +PYEOF +} colcon_test_these_packages() { THIS_DIR=$(pwd) cd ${CATKIN_WS_PATH} colcon_build_no_deps $1 source install/setup.bash && \ colcon test --packages-select $1 for pkg in $1; do - colcon test-result --verbose --test-result-base "build/$pkg" + if ! colcon test-result --test-result-base "build/$pkg/test_results"; then + _show_test_failures "build/$pkg/test_results" + fi done cd $THIS_DIR @@ -135,6 +155,56 @@ er_pylint_sorted_here() { er_pylint_here | sort -V | grep -v "\*\*\*\*\*\*\*\*"; er_pylint_single_file() { check_pylintrc; check_apt_package "pylint"; pylint --rcfile /tmp/pylintrc --max-line-length ${LINTER_MAX_LINE_LENGTH} $1; } er_pylint_single_file_sorted() { er_pylint_single_file $1 | sort -V | grep -v "\*\*\*\*\*\*\*\*"; return ${PIPESTATUS[0]}; } er_ruff_here() { ruff check; } + + +# reproduce ci locally +DEFAULT_CI_CONTAINER_NAME="er_ci_reproduced_testing_env" +reproduce_ci() { + local token="${ER_SETUP_TOKEN:-}" + local args=("$@") + local has_token="false" + local CI_BRANCH="ERD-1633_reproduce_ci_locally" + for arg in "${args[@]}"; do + if [ "${arg}" = "--gh-token" ] || [ "${arg}" = "-t" ]; then + has_token="true" + break + fi + done + if [ "${has_token}" = "false" ]; then + if [ -z "${token}" ]; then + echo -e "${Red}Error: --gh-token not provided and ER_SETUP_TOKEN is not set${Color_Off}" + return 1 + fi + args=("--gh-token" "${token}" "${args[@]}") + fi + local wrapper_script + wrapper_script=$(curl -fSL https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/${CI_BRANCH}/bin/reproduce_ci.sh) || { + echo -e "${Red}Error: Failed to fetch reproduce_ci.sh wrapper from er_build_tools main branch${Color_Off}" + return 1 + } + bash <(echo "${wrapper_script}") "${args[@]}" +} +repull_and_rerun_ci_tests() { + local container_name="${1:-${DEFAULT_CI_CONTAINER_NAME}}" + if ! docker ps --filter "name=^${container_name}$" --format '{{.Names}}' | grep -q .; then + echo -e "${Red}Error: Container '${container_name}' is not running${Color_Off}" + return 1 + fi + docker exec "${container_name}" bash /tmp/ci_repull_and_retest.sh +} +remove_ci_container() { + local container_name="${1:-${DEFAULT_CI_CONTAINER_NAME}}" + if ! docker ps -a --filter "name=^${container_name}$" --format '{{.Names}}' | grep -q .; then + echo -e "${Yellow}Container '${container_name}' does not exist${Color_Off}" + return 0 + fi + echo "Stopping and removing container '${container_name}'..." + docker rm -f "${container_name}" + echo -e "${Green}Container '${container_name}' removed${Color_Off}" +} + + + er_python_linters_here() { local ret=0 echo er_pylint_sorted_here || ret=$? diff --git a/README.md b/README.md index 3041107..5f56e6a 100644 --- a/README.md +++ b/README.md @@ -1 +1,105 @@ # er_build_tools + +Public build tools and utilities for Extend Robotics repositories. + +## reproduce_ci.sh — Reproduce CI Locally + +When CI fails, debugging requires pushing commits and waiting for results. This script reproduces the exact CI environment locally in a persistent Docker container, so you can debug interactively. + +It creates a Docker container using the same image as CI, clones your repo and its dependencies, builds everything, and optionally runs tests — mirroring the steps in `setup_and_build_ros_ws.yml`. + +### Quick Start + +```bash +bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \ + --gh-token "$GH_TOKEN" \ + --repo https://github.com/Extend-Robotics/er_interface \ + --only-needed-deps +``` + +### Requirements + +- Docker installed and running +- A GitHub token (`--gh-token`) with access to Extend-Robotics private repos + +### Options + +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--gh-token` | `-t` | *required* | GitHub token with access to private repos | +| `--repo` | `-r` | *required* | Repository URL to test | +| `--branch` | `-b` | `main` | Branch or commit SHA to test | +| `--only-needed-deps` | | off | Only build deps needed by the repo under test (faster) | +| `--skip-tests` | | off | Skip running colcon tests | +| `--image` | `-i` | `rostooling/setup-ros-docker:ubuntu-focal-ros-noetic-desktop-latest` | Docker image | +| `--container-name` | `-n` | `er_ci_reproduced_testing_env` | Docker container name | +| `--deps-file` | `-d` | `deps.repos` | Path to deps file in the repo | +| `--graphical` | `-g` | `true` | Enable X11/NVIDIA forwarding | +| `--additional-command` | `-c` | | Extra command to run after build/test | +| `--scripts-branch` | | `main` | Branch of `er_build_tools_internal` to fetch scripts from | + +### Examples + +Test a specific branch with all deps: + +```bash +bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \ + --gh-token "$GH_TOKEN" \ + --repo https://github.com/Extend-Robotics/er_interface \ + --branch my-feature-branch +``` + +Build only, skip tests, no graphical forwarding: + +```bash +bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \ + --gh-token "$GH_TOKEN" \ + --repo https://github.com/Extend-Robotics/er_interface \ + --only-needed-deps \ + --skip-tests \ + --graphical false +``` + +Test with feature branches of both `er_build_tools` and `er_build_tools_internal` (useful when developing the CI scripts themselves): + +```bash +bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/my-feature-branch-in-er-build-tools/bin/reproduce_ci.sh) \ + --gh-token "$GH_TOKEN" \ + --repo https://github.com/Extend-Robotics/er_interface \ + --scripts-branch my-feature-branch-in-er-build-tools-internal \ + --only-needed-deps +``` + +Run xacro lint after build (like er_interface CI does): + +```bash +bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \ + --gh-token "$GH_TOKEN" \ + --repo https://github.com/Extend-Robotics/er_interface \ + --only-needed-deps \ + --additional-command "python3 ros_ws/src/er_interface/er_interface/src/er_interface/xacro_lint.py" +``` + +### After the Script Completes + +The container stays running. You can enter it to debug interactively: + +```bash +docker exec -it er_ci_reproduced_testing_env bash +``` + +The workspace is at `/ros_ws` inside the container. + +To clean up: + +```bash +docker rm -f er_ci_reproduced_testing_env +``` + +### Troubleshooting + +**Container already exists** — Remove it first: `docker rm -f er_ci_reproduced_testing_env` + +**404 when fetching scripts** — Check that your `--gh-token` has access to `er_build_tools_internal`, and that the `--scripts-branch` exists. + +**`DISPLAY` error with graphical forwarding** — Either set `DISPLAY` (e.g. via X11 forwarding) or pass `--graphical false`. diff --git a/bin/reproduce_ci.sh b/bin/reproduce_ci.sh new file mode 100755 index 0000000..1037807 --- /dev/null +++ b/bin/reproduce_ci.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# SKIP_CHECK +set -euo pipefail + +# Public wrapper: fetches the real CI reproduction scripts from er_build_tools_internal (private) +# and runs them. This script is the entry point for remote execution via: +# bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \ +# --gh-token ghp_xxx --repo https://github.com/extend-robotics/er_interface + +gh_token="" +scripts_branch="main" +for i in $(seq 1 $#); do + arg="${!i}" + if [ "${arg}" = "--gh-token" ] || [ "${arg}" = "-t" ]; then + next=$((i + 1)) + if [ "${next}" -gt "$#" ]; then + echo "Error: ${arg} requires a value" + exit 1 + fi + gh_token="${!next}" + elif [ "${arg}" = "--scripts-branch" ] || [ "${arg}" = "--scripts_branch" ]; then + next=$((i + 1)) + if [ "${next}" -gt "$#" ]; then + echo "Error: ${arg} requires a value" + exit 1 + fi + scripts_branch="${!next}" + fi +done + +if [ -z "${gh_token}" ]; then + echo "Error: --gh-token is required to fetch scripts from er_build_tools_internal" + echo "Usage: bash <(curl -Ls ...) --gh-token --repo [options]" + exit 1 +fi + +SCRIPT_DIR="/tmp/er_reproduce_ci" +echo "Creating script directory: ${SCRIPT_DIR}" +mkdir -p "${SCRIPT_DIR}" + +RAW_URL="https://raw.githubusercontent.com/Extend-Robotics/er_build_tools_internal/refs/heads/${scripts_branch}" + +fetch_script() { + local script_name="$1" + local source_url="${RAW_URL}/bin/${script_name}" + local destination="${SCRIPT_DIR}/${script_name}" + + echo "Fetching ${script_name}:" + echo " From: ${source_url}" + echo " To: ${destination}" + + local http_code + http_code=$(curl -fSL -w "%{http_code}" -H "Authorization: token ${gh_token}" "${source_url}" -o "${destination}") || { + echo " FAILED (HTTP ${http_code})" + echo "" + echo "Error: Failed to fetch ${script_name}" + echo " Check that the branch '${scripts_branch}' exists in er_build_tools_internal" + echo " Check that your --gh-token has access to Extend-Robotics/er_build_tools_internal" + exit 1 + } + echo " OK (HTTP ${http_code})" + + if [ ! -s "${destination}" ]; then + echo "Error: Downloaded file is empty: ${destination}" + exit 1 + fi +} + +echo "Fetching scripts from er_build_tools_internal (branch: ${scripts_branch})..." +fetch_script "reproduce_ci.sh" +fetch_script "ci_workspace_setup.sh" +fetch_script "ci_repull_and_retest.sh" + +chmod +x "${SCRIPT_DIR}/reproduce_ci.sh" +echo "" +echo "Running ${SCRIPT_DIR}/reproduce_ci.sh..." +"${SCRIPT_DIR}/reproduce_ci.sh" "$@"