From 5ce11296d596d7af944f115e9b090e8b0e5ab3f2 Mon Sep 17 00:00:00 2001 From: Melanie Li Date: Thu, 19 Mar 2026 02:18:07 +0000 Subject: [PATCH] fix: add --dangerously-skip-permissions for CI/CD environments In CI/CD environments (CodeBuild, GitHub Actions, etc.), Claude Code has no user configuration (~/.claude/settings.json). Without permission bypass, Claude Code prompts for confirmation before executing Bash commands, which in -p (print) mode causes the tool call to be skipped. This results in: - Agent never executing skill scripts (e.g., python3 scripts/check.py) - Functional scores being artificially low (agent can't use skill tools) - Identical with-skill and without-skill results Fix: Add --dangerously-skip-permissions to both with-skill and without-skill Claude Code invocations. This is safe because: 1. Eval runs in isolated temp workspaces (no access to real data) 2. The agent is already sandboxed by --allowedTools 3. CI/CD environments are ephemeral containers This matches the Claude Code documentation recommendation for sandboxed environments without internet access. --- skill_eval/agent_runner.py | 2 ++ tests/test_agent_runner.py | 1 + 2 files changed, 3 insertions(+) diff --git a/skill_eval/agent_runner.py b/skill_eval/agent_runner.py index 9dd92ba..d60d3a5 100644 --- a/skill_eval/agent_runner.py +++ b/skill_eval/agent_runner.py @@ -140,6 +140,7 @@ def _build_cmd_with_skill(self, prompt: str, skill_path: str) -> list[str]: skill_content = self._read_skill_content(skill_path) cmd = [ self.CLI_NAME, "-p", prompt, + "--dangerously-skip-permissions", "--allowedTools", "Read", "Glob", "Grep", "Bash", "Write", "Edit", ] if skill_content: @@ -161,6 +162,7 @@ def _build_cmd_without_skill(self, prompt: str) -> list[str]: """Build claude CLI argument list for running WITHOUT a skill.""" return [ self.CLI_NAME, "-p", prompt, + "--dangerously-skip-permissions", "--allowedTools", "Read", "Glob", "Grep", "Bash", "Write", "Edit", ] diff --git a/tests/test_agent_runner.py b/tests/test_agent_runner.py index 81b5736..7193f3f 100644 --- a/tests/test_agent_runner.py +++ b/tests/test_agent_runner.py @@ -405,6 +405,7 @@ def test_build_cmd_without_skill(self): assert "claude" == cmd[0] assert "-p" in cmd assert "test prompt" in cmd + assert "--dangerously-skip-permissions" in cmd assert "--allowedTools" in cmd def test_build_cmd_with_skill_no_skill_md(self, tmp_path):