Erd 1633 reproduce ci locally

.helper_bash_functions

@@ -26,13 +26,33 @@ colcon_build() { START_DIR=$(pwd) && \
 }
 colcon_build_this() { colcon_build "$(find_cmake_project_names_from_dir .)"; }
 rosdep_install() { rosdep install --from-paths src --ignore-src -r -y ; }
+_show_test_failures() {
+    python3 - "$@" <<'PYEOF'
+import sys, xml.etree.ElementTree as ET
+from pathlib import Path
+for d in sys.argv[1:]:
+    for p in sorted(Path(d).rglob("*.xml")):
+        for tc in ET.parse(p).iter("testcase"):
+            for f in list(tc.iter("failure")) + list(tc.iter("error")):
+                tag = "FAIL" if f.tag == "failure" else "ERROR"
+                print(f"\n  {tag}: {tc.get('classname', '')}.{tc.get('name', '')}")
+                if f.text:
+                    lines = f.text.strip().splitlines()
+                    for l in lines[:20]:
+                        print(f"    {l}")
+                    if len(lines) > 20:
+                        print(f"    ... ({len(lines) - 20} more lines)")
+PYEOF
+}
 colcon_test_these_packages() { THIS_DIR=$(pwd)
                                cd ${CATKIN_WS_PATH}
                                colcon_build_no_deps $1
                                source install/setup.bash && \
                                colcon test --packages-select $1
                                for pkg in $1; do
-                                   colcon test-result --verbose --test-result-base "build/$pkg"
+                                   if ! colcon test-result --test-result-base "build/$pkg/test_results"; then
+                                       _show_test_failures "build/$pkg/test_results"
+                                   fi
                                done
                                cd $THIS_DIR
 
@@ -135,6 +155,56 @@ er_pylint_sorted_here() { er_pylint_here | sort -V | grep -v "\*\*\*\*\*\*\*\*";
 er_pylint_single_file() { check_pylintrc; check_apt_package "pylint";  pylint --rcfile /tmp/pylintrc --max-line-length ${LINTER_MAX_LINE_LENGTH} $1; }
 er_pylint_single_file_sorted() { er_pylint_single_file $1 | sort -V | grep -v "\*\*\*\*\*\*\*\*"; return ${PIPESTATUS[0]}; }
 er_ruff_here() { ruff check; }
+
+
+# reproduce ci locally
+DEFAULT_CI_CONTAINER_NAME="er_ci_reproduced_testing_env"
+reproduce_ci() {
+    local token="${ER_SETUP_TOKEN:-}"
+    local args=("$@")
+    local has_token="false"
+    local CI_BRANCH="ERD-1633_reproduce_ci_locally"
+    for arg in "${args[@]}"; do
+        if [ "${arg}" = "--gh-token" ] || [ "${arg}" = "-t" ]; then
+            has_token="true"
+            break
+        fi
+    done
+    if [ "${has_token}" = "false" ]; then
+        if [ -z "${token}" ]; then
+            echo -e "${Red}Error: --gh-token not provided and ER_SETUP_TOKEN is not set${Color_Off}"
+            return 1
+        fi
+        args=("--gh-token" "${token}" "${args[@]}")
+    fi
+    local wrapper_script
+    wrapper_script=$(curl -fSL https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/${CI_BRANCH}/bin/reproduce_ci.sh) || {
+        echo -e "${Red}Error: Failed to fetch reproduce_ci.sh wrapper from er_build_tools main branch${Color_Off}"
+        return 1
+    }
+    bash <(echo "${wrapper_script}") "${args[@]}"
+}
+repull_and_rerun_ci_tests() {
+    local container_name="${1:-${DEFAULT_CI_CONTAINER_NAME}}"
+    if ! docker ps --filter "name=^${container_name}$" --format '{{.Names}}' | grep -q .; then
+        echo -e "${Red}Error: Container '${container_name}' is not running${Color_Off}"
+        return 1
+    fi
+    docker exec "${container_name}" bash /tmp/ci_repull_and_retest.sh
+}
+remove_ci_container() {
+    local container_name="${1:-${DEFAULT_CI_CONTAINER_NAME}}"
+    if ! docker ps -a --filter "name=^${container_name}$" --format '{{.Names}}' | grep -q .; then
+        echo -e "${Yellow}Container '${container_name}' does not exist${Color_Off}"
+        return 0
+    fi
+    echo "Stopping and removing container '${container_name}'..."
+    docker rm -f "${container_name}"
+    echo -e "${Green}Container '${container_name}' removed${Color_Off}"
+}
+
+
+
 er_python_linters_here() { local ret=0
                            echo
                            er_pylint_sorted_here || ret=$?

README.md

@@ -1 +1,105 @@
 # er_build_tools
+
+Public build tools and utilities for Extend Robotics repositories.
+
+## reproduce_ci.sh — Reproduce CI Locally
+
+When CI fails, debugging requires pushing commits and waiting for results. This script reproduces the exact CI environment locally in a persistent Docker container, so you can debug interactively.
+
+It creates a Docker container using the same image as CI, clones your repo and its dependencies, builds everything, and optionally runs tests — mirroring the steps in `setup_and_build_ros_ws.yml`.
+
+### Quick Start
+
+```bash
+bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \
+  --gh-token "$GH_TOKEN" \
+  --repo https://github.com/Extend-Robotics/er_interface \
+  --only-needed-deps
+```
+
+### Requirements
+
+- Docker installed and running
+- A GitHub token (`--gh-token`) with access to Extend-Robotics private repos
+
+### Options
+
+| Flag | Short | Default | Description |
+|------|-------|---------|-------------|
+| `--gh-token` | `-t` | *required* | GitHub token with access to private repos |
+| `--repo` | `-r` | *required* | Repository URL to test |
+| `--branch` | `-b` | `main` | Branch or commit SHA to test |
+| `--only-needed-deps` | | off | Only build deps needed by the repo under test (faster) |
+| `--skip-tests` | | off | Skip running colcon tests |
+| `--image` | `-i` | `rostooling/setup-ros-docker:ubuntu-focal-ros-noetic-desktop-latest` | Docker image |
+| `--container-name` | `-n` | `er_ci_reproduced_testing_env` | Docker container name |
+| `--deps-file` | `-d` | `deps.repos` | Path to deps file in the repo |
+| `--graphical` | `-g` | `true` | Enable X11/NVIDIA forwarding |
+| `--additional-command` | `-c` | | Extra command to run after build/test |
+| `--scripts-branch` | | `main` | Branch of `er_build_tools_internal` to fetch scripts from |
+
+### Examples
+
+Test a specific branch with all deps:
+
+```bash
+bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \
+  --gh-token "$GH_TOKEN" \
+  --repo https://github.com/Extend-Robotics/er_interface \
+  --branch my-feature-branch
+```
+
+Build only, skip tests, no graphical forwarding:
+
+```bash
+bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \
+  --gh-token "$GH_TOKEN" \
+  --repo https://github.com/Extend-Robotics/er_interface \
+  --only-needed-deps \
+  --skip-tests \
+  --graphical false
+```
+
+Test with feature branches of both `er_build_tools` and `er_build_tools_internal` (useful when developing the CI scripts themselves):
+
+```bash
+bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/my-feature-branch-in-er-build-tools/bin/reproduce_ci.sh) \
+  --gh-token "$GH_TOKEN" \
+  --repo https://github.com/Extend-Robotics/er_interface \
+  --scripts-branch my-feature-branch-in-er-build-tools-internal \
+  --only-needed-deps
+```
+
+Run xacro lint after build (like er_interface CI does):
+
+```bash
+bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \
+  --gh-token "$GH_TOKEN" \
+  --repo https://github.com/Extend-Robotics/er_interface \
+  --only-needed-deps \
+  --additional-command "python3 ros_ws/src/er_interface/er_interface/src/er_interface/xacro_lint.py"
+```
+
+### After the Script Completes
+
+The container stays running. You can enter it to debug interactively:
+
+```bash
+docker exec -it er_ci_reproduced_testing_env bash
+```
+
+The workspace is at `/ros_ws` inside the container.
+
+To clean up:
+
+```bash
+docker rm -f er_ci_reproduced_testing_env
+```
+
+### Troubleshooting
+
+**Container already exists** — Remove it first: `docker rm -f er_ci_reproduced_testing_env`
+
+**404 when fetching scripts** — Check that your `--gh-token` has access to `er_build_tools_internal`, and that the `--scripts-branch` exists.
+
+**`DISPLAY` error with graphical forwarding** — Either set `DISPLAY` (e.g. via X11 forwarding) or pass `--graphical false`.

bin/reproduce_ci.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# SKIP_CHECK
+set -euo pipefail
+
+# Public wrapper: fetches the real CI reproduction scripts from er_build_tools_internal (private)
+# and runs them. This script is the entry point for remote execution via:
+#   bash <(curl -Ls https://raw.githubusercontent.com/Extend-Robotics/er_build_tools/refs/heads/main/bin/reproduce_ci.sh) \
+#     --gh-token ghp_xxx --repo https://github.com/extend-robotics/er_interface
+
+gh_token=""
+scripts_branch="main"
+for i in $(seq 1 $#); do
+    arg="${!i}"
+    if [ "${arg}" = "--gh-token" ] || [ "${arg}" = "-t" ]; then
+        next=$((i + 1))
+        if [ "${next}" -gt "$#" ]; then
+            echo "Error: ${arg} requires a value"
+            exit 1
+        fi
+        gh_token="${!next}"
+    elif [ "${arg}" = "--scripts-branch" ] || [ "${arg}" = "--scripts_branch" ]; then
+        next=$((i + 1))
+        if [ "${next}" -gt "$#" ]; then
+            echo "Error: ${arg} requires a value"
+            exit 1
+        fi
+        scripts_branch="${!next}"
+    fi
+done
+
+if [ -z "${gh_token}" ]; then
+    echo "Error: --gh-token is required to fetch scripts from er_build_tools_internal"
+    echo "Usage: bash <(curl -Ls ...) --gh-token <token> --repo <repo_url> [options]"
+    exit 1
+fi
+
+SCRIPT_DIR="/tmp/er_reproduce_ci"
+echo "Creating script directory: ${SCRIPT_DIR}"
+mkdir -p "${SCRIPT_DIR}"
+
+RAW_URL="https://raw.githubusercontent.com/Extend-Robotics/er_build_tools_internal/refs/heads/${scripts_branch}"
+
+fetch_script() {
+    local script_name="$1"
+    local source_url="${RAW_URL}/bin/${script_name}"
+    local destination="${SCRIPT_DIR}/${script_name}"
+
+    echo "Fetching ${script_name}:"
+    echo "  From: ${source_url}"
+    echo "  To:   ${destination}"
+
+    local http_code
+    http_code=$(curl -fSL -w "%{http_code}" -H "Authorization: token ${gh_token}" "${source_url}" -o "${destination}") || {
+        echo "  FAILED (HTTP ${http_code})"
+        echo ""
+        echo "Error: Failed to fetch ${script_name}"
+        echo "  Check that the branch '${scripts_branch}' exists in er_build_tools_internal"
+        echo "  Check that your --gh-token has access to Extend-Robotics/er_build_tools_internal"
+        exit 1
+    }
+    echo "  OK (HTTP ${http_code})"
+
+    if [ ! -s "${destination}" ]; then
+        echo "Error: Downloaded file is empty: ${destination}"
+        exit 1
+    fi
+}
+
+echo "Fetching scripts from er_build_tools_internal (branch: ${scripts_branch})..."
+fetch_script "reproduce_ci.sh"
+fetch_script "ci_workspace_setup.sh"
+fetch_script "ci_repull_and_retest.sh"
+
+chmod +x "${SCRIPT_DIR}/reproduce_ci.sh"
+echo ""
+echo "Running ${SCRIPT_DIR}/reproduce_ci.sh..."
+"${SCRIPT_DIR}/reproduce_ci.sh" "$@"