lots of silent failures

Author: Build Tools

bin/ci_tool/ci_fix.py

@@ -113,6 +113,10 @@ def read_container_state(container_name):
     try:
         return json.loads(result.stdout)
     except json.JSONDecodeError:
+        console.print(
+            f"[yellow]State file exists but contains invalid JSON: "
+            f"{result.stdout[:200]}[/yellow]"
+        )
         return None
 
 
@@ -355,23 +359,35 @@ def run_claude_streamed(container_name, prompt):
     """Run Claude non-interactively with stream-json output."""
     escaped_prompt = prompt.replace("'", "'\\''")
     claude_command = (
+        f"set -o pipefail && "
         f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions "
         f"-p '{escaped_prompt}' --verbose --output-format stream-json "
         f"2>{CLAUDE_STDERR_LOG} | ci_fix_display"
     )
-    docker_exec(container_name, claude_command, tty=True, check=False, quiet=True)
+    result = docker_exec(container_name, claude_command, tty=True, check=False, quiet=True)
+    if result.returncode != 0:
+        console.print(
+            f"[yellow]Claude exited with code {result.returncode} — "
+            f"check {CLAUDE_STDERR_LOG} inside the container for details[/yellow]"
+        )
 
 
 def run_claude_resumed(container_name, session_id, prompt):
     """Resume a Claude session with a new prompt, streaming output."""
     escaped_prompt = prompt.replace("'", "'\\''")
     claude_command = (
+        f"set -o pipefail && "
         f"cd /ros_ws && IS_SANDBOX=1 claude --dangerously-skip-permissions "
         f"--resume '{session_id}' -p '{escaped_prompt}' "
         f"--verbose --output-format stream-json "
         f"2>{CLAUDE_STDERR_LOG} | ci_fix_display"
     )
-    docker_exec(container_name, claude_command, tty=True, check=False, quiet=True)
+    result = docker_exec(container_name, claude_command, tty=True, check=False, quiet=True)
+    if result.returncode != 0:
+        console.print(
+            f"[yellow]Claude exited with code {result.returncode} — "
+            f"check {CLAUDE_STDERR_LOG} inside the container for details[/yellow]"
+        )
 
 
 def prompt_user_for_feedback():
@@ -431,7 +447,7 @@ def run_claude_workflow(container_name, ci_run_info):
 
         # Fix phase (resume session)
         state = read_container_state(container_name)
-        session_id = state["session_id"] if state else None
+        session_id = state.get("session_id") if state else None
         if session_id:
             console.print(
                 "\n[bold cyan]Resuming Claude "
@@ -540,13 +556,17 @@ def fix_ci(_args):
         console.print(
             "[dim]You are now in an interactive Claude session[/dim]\n"
         )
-        docker_exec(
+        result = docker_exec(
             container_name,
             "cd /ros_ws && IS_SANDBOX=1 claude "
             "--dangerously-skip-permissions "
             f'--resume "{resume_session_id}"',
             interactive=True, check=False,
         )
+        if result.returncode != 0:
+            console.print(
+                f"[yellow]Claude exited with code {result.returncode}[/yellow]"
+            )
     else:
         run_claude_workflow(
             container_name, session.get("ci_run_info")

bin/ci_tool/ci_reproduce.py

@@ -199,10 +199,20 @@ def _docker_create_and_start(
     console.print(f"  [green]\u2713[/green] Container '{container_name}' started")
 
 
+def _container_path_exists(container_name, path):
+    """Check if a path exists inside a container."""
+    result = subprocess.run(
+        ["docker", "exec", container_name, "test", "-e", path],
+        check=False,
+    )
+    return result.returncode == 0
+
+
 def _docker_exec_workspace_setup(container_name):
     """Run ci_workspace_setup.sh inside the container.
 
-    Handles KeyboardInterrupt gracefully by letting the container keep running.
+    Raises RuntimeError if setup fails before the build completes.
+    Warns (but continues) if tests fail after a successful build.
     """
     console.print("\n  Running CI workspace setup inside container...")
     try:
@@ -215,14 +225,27 @@ def _docker_exec_workspace_setup(container_name):
             "\n[yellow]Interrupted during workspace setup "
             "-- container is still running with partial setup[/yellow]"
         )
+        raise
+
+    if result.returncode == 0:
         return
 
-    if result.returncode != 0:
-        console.print(
-            f"\n[yellow]Workspace setup exited with code {result.returncode} "
-            f"(expected if tests failed)[/yellow]"
+    # Non-zero exit: distinguish setup failure from test failure by checking
+    # whether the build actually completed (/ros_ws/install only exists after
+    # a successful colcon build).
+    build_completed = _container_path_exists(container_name, "/ros_ws/install")
+
+    if not build_completed:
+        raise RuntimeError(
+            f"Workspace setup failed (exit code {result.returncode}). "
+            f"The build did not complete — check the output above for errors."
         )
 
+    console.print(
+        f"\n[yellow]Tests exited with code {result.returncode} "
+        f"(build succeeded, test failures are expected)[/yellow]"
+    )
+
 
 def prompt_for_repo_and_branch():
     """Ask user for repository URL and branch name interactively.

bin/ci_tool/claude_setup.py

@@ -34,9 +34,11 @@ def install_claude_in_container(container_name):
 
 
 def install_fzf_in_container(container_name):
-    """Install fzf in the container."""
+    """Install fzf in the container (non-critical)."""
     console.print("[cyan]Installing fzf in container...[/cyan]")
-    docker_exec(container_name, "apt-get update && apt-get install -y fzf", check=False)
+    result = docker_exec(container_name, "apt-get update && apt-get install -y fzf", check=False)
+    if result.returncode != 0:
+        console.print("[yellow]fzf installation failed (non-critical) — continuing[/yellow]")
 
 
 def install_python_deps_in_container(container_name):
@@ -187,7 +189,10 @@ def save_package_list(container_name):
         quiet=True,
     )
     if result.returncode != 0:
-        console.print("[yellow]Could not save package list (colcon list failed)[/yellow]")
+        console.print(
+            "[yellow]Could not save package list (colcon list failed). "
+            "The 'rerun_tests' helper will not work.[/yellow]"
+        )
 
 
 def inject_rerun_tests_function(container_name):
@@ -267,7 +272,7 @@ def get_host_git_config(key):
 
 def configure_git_in_container(container_name):
     """Set up git identity, token-based auth, and gh CLI auth in the container."""
-    gh_token = os.environ.get("GH_TOKEN", "")
+    gh_token = os.environ.get("GH_TOKEN") or os.environ.get("ER_SETUP_TOKEN") or ""
 
     git_user_name = get_host_git_config("user.name")
     git_user_email = get_host_git_config("user.email")
@@ -276,20 +281,26 @@ def configure_git_in_container(container_name):
     docker_exec(container_name, f'git config --global user.name "{git_user_name}"')
     docker_exec(container_name, f'git config --global user.email "{git_user_email}"')
 
-    if gh_token:
-        docker_exec(
-            container_name,
-            f'git config --global url."https://{gh_token}@github.com/"'
-            f'.insteadOf "https://github.com/"',
-            quiet=True,
+    if not gh_token:
+        console.print(
+            "[yellow]No GH_TOKEN or ER_SETUP_TOKEN found — "
+            "git auth and gh CLI will not be configured in container[/yellow]"
         )
-        install_and_auth_gh_cli(container_name, gh_token)
+        return
+
+    docker_exec(
+        container_name,
+        f'git config --global url."https://{gh_token}@github.com/"'
+        f'.insteadOf "https://github.com/"',
+        quiet=True,
+    )
+    install_and_auth_gh_cli(container_name, gh_token)
 
 
 def install_and_auth_gh_cli(container_name, gh_token):
     """Install gh CLI and authenticate with the provided token."""
     console.print("[cyan]Installing gh CLI in container...[/cyan]")
-    docker_exec(container_name, (
+    install_result = docker_exec(container_name, (
         "type gh >/dev/null 2>&1 || ("
         "curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg "
         "| dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg "
@@ -300,13 +311,25 @@ def install_and_auth_gh_cli(container_name, gh_token):
         "| tee /etc/apt/sources.list.d/github-cli-stable.list > /dev/null "
         "&& apt-get update && apt-get install -y gh)"
     ), check=False)
+    if install_result.returncode != 0:
+        console.print(
+            "[yellow]gh CLI installation failed — "
+            "Claude will not be able to interact with GitHub from inside the container[/yellow]"
+        )
+        return
+
     console.print("[cyan]Authenticating gh CLI...[/cyan]")
-    docker_exec(
+    auth_result = docker_exec(
         container_name,
         f'echo "{gh_token}" | gh auth login --with-token',
         check=False,
         quiet=True,
     )
+    if auth_result.returncode != 0:
+        console.print(
+            "[yellow]gh CLI authentication failed — "
+            "Claude will not be able to interact with GitHub from inside the container[/yellow]"
+        )
 
 
 def is_claude_installed_in_container(container_name):

bin/ci_tool/containers.py

@@ -35,6 +35,8 @@ def container_exists(container_name=DEFAULT_CONTAINER_NAME):
         ["docker", "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}"],
         capture_output=True, text=True, check=False,
     )
+    if result.returncode != 0:
+        raise RuntimeError(f"docker ps failed: {result.stderr.strip()}")
     return container_name in result.stdout.strip()
 
 
@@ -44,6 +46,8 @@ def container_is_running(container_name=DEFAULT_CONTAINER_NAME):
         ["docker", "ps", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}"],
         capture_output=True, text=True, check=False,
     )
+    if result.returncode != 0:
+        raise RuntimeError(f"docker ps failed: {result.stderr.strip()}")
     return container_name in result.stdout.strip()
 
 
@@ -65,6 +69,8 @@ def list_ci_containers():
          "--format", "{{.Names}}\t{{.Status}}"],
         capture_output=True, text=True, check=False,
     )
+    if result.returncode != 0:
+        raise RuntimeError(f"docker ps failed: {result.stderr.strip()}")
     if not result.stdout.strip():
         return []
     containers = []

bin/ci_tool/display_progress.py

@@ -52,7 +52,10 @@ def read_existing_attempt_count():
     try:
         with open(STATE_FILE, encoding="utf-8") as state_file:
             return json.load(state_file).get("attempt_count", 0)
-    except (FileNotFoundError, json.JSONDecodeError, KeyError):
+    except FileNotFoundError:
+        return 0
+    except (json.JSONDecodeError, KeyError) as error:
+        console.print(f"[yellow]State file corrupt, resetting attempt count: {error}[/yellow]")
         return 0
 
 
@@ -120,7 +123,7 @@ def print_session_summary(session_id, start_time):
             console.print("[yellow]Claude stderr output:[/yellow]")
             console.print(stderr_content)
     except FileNotFoundError:
-        pass
+        console.print(f"[dim]No stderr log at {CLAUDE_STDERR_LOG}[/dim]")
 
 
 def main():
@@ -145,6 +148,7 @@ def main():
                 try:
                     event = json.loads(line)
                 except json.JSONDecodeError:
+                    sys.stderr.write(f"  {line}\n")
                     continue
 
                 event_session_id = handle_event(event, start_time)
@@ -155,7 +159,7 @@ def main():
 
     except KeyboardInterrupt:
         phase = "interrupted"
-    except Exception:  # pylint: disable=broad-except
+    except (IOError, ValueError, UnicodeDecodeError):
         console.print(
             f"\n[red]Display processor error:[/red]\n{traceback.format_exc()}"
         )