Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ poetry run inferedgelab agent-runtime-report \
--orchestration-summary examples/agent_runtime/agent_3_orchestration_summary.json \
--guard-analysis examples/agent_runtime/aiguard_runtime_guard_analysis.json
bash scripts/smoke_agent_runtime_remote_paths.sh --output-dir reports/agent_runtime_remote_paths
bash scripts/smoke_agent_runtime_edgeenv_preservation.sh --output-dir reports/agent_runtime_edgeenv_preservation
poetry run inferedgelab export-demo-evidence --output reports/studio_demo_evidence.md
```

Expand All @@ -188,7 +189,9 @@ Additional report paths:
- The bundled agent evidence is a synthetic sustained high-load 3-agent scenario for local review.
- Optional `--runtime-result <path>` input adds Runtime-side `runtime_health_snapshot`, `runtime_events`, and `runtime_operation_summary` context to the same Lab report.
- Optional `--remote-dispatch <path>` input adds file-based worker selection, retry/fallback plan, and remote execution starter context when an Orchestrator `inferedge-remote-dispatch-result-v1` JSON is available.
- Optional `--edgeenv-run-show <path>` input adds EdgeEnv local run preservation context when an EdgeEnv `runs show` JSON is available.
- `scripts/smoke_agent_runtime_remote_paths.sh` reproduces both committed remote dispatch starter paths: plan-only worker selection and fallback-recovered starter execution.
- `scripts/smoke_agent_runtime_edgeenv_preservation.sh` gates the committed EdgeEnv run-show fixture so the `Runtime Intelligence EdgeEnv Preservation` report section cannot disappear silently.
- Remote dispatch remains starter evidence for local-first review; it does not claim production remote execution.

Remote dispatch starter boundary:
Expand Down
15 changes: 15 additions & 0 deletions docs/portfolio/agent_runtime_reliability_report.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ Committed lightweight fixtures:
- `examples/agent_runtime/aiguard_runtime_guard_analysis.json`
- `examples/agent_runtime/remote_dispatch_plan_only.json`
- `examples/agent_runtime/remote_dispatch_fallback_recovered.json`
- `examples/agent_runtime/edgeenv_run_show_runtime_operation.json`

Generate a Markdown report:

Expand All @@ -53,6 +54,20 @@ fallback-recovered starter execution. Both reports preserve
`remote_runtime_summary_boundary=remote dispatch starter evidence only`; they
do not start remote workers or claim production remote execution.

Reproduce the committed EdgeEnv run preservation report path:

```bash
bash scripts/smoke_agent_runtime_edgeenv_preservation.sh \
--output-dir reports/agent_runtime_edgeenv_preservation
```

The smoke verifies that the Lab-owned Markdown/JSON report keeps the
`Runtime Intelligence EdgeEnv Preservation` section, EdgeEnv run ID, Runtime
operation summary schema, and `comparability_role=supplemental_evidence_not_gate`
visible from a lightweight `runs show` fixture. This is local registry
preservation evidence, not a production telemetry database or deployment
decision override.

Current starter coverage:

- file-based worker registry and task request ingestion
Expand Down
69 changes: 69 additions & 0 deletions examples/agent_runtime/edgeenv_run_show_runtime_operation.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"created_at": "2026-05-29T09:47:14.594755Z",
"metrics": {
"latency_mean_ms": 28.0,
"latency_p50_ms": 28.0,
"latency_p95_ms": 36.0,
"latency_p99_ms": 36.0,
"throughput_fps": 30.0
},
"model": {
"model_format": "runtime-result-json",
"model_hash": "acb5f50c195d2feef052cb483fb8b3746ea8cd64a28aa8a44e9dfe2358b3c487",
"model_name": "entrypoint-runtime-operation-result",
"model_path": "examples/agent_runtime/runtime_operation_result.json",
"model_version": "v1"
},
"protocol": {
"batch_size": 1,
"include_postprocess": false,
"include_preprocess": false,
"input_dtype": "json",
"input_shape": [
1
],
"precision": "evidence",
"repeat_runs": 1,
"task": "runtime-operation-evidence",
"warmup_runs": 0
},
"result_path": "examples/agent_runtime/.edgeenv/runs/run-fixture-edgeenv-operation-0001/result.json",
"run_id": "run-fixture-edgeenv-operation-0001",
"runtime": {
"execution_provider": "local-python",
"runtime": "entrypoint-smoke"
},
"runtime_operation_summary": {
"deadline_missed": false,
"decision_owner": "lab",
"error_category": "runtime_timeout_observed",
"evidence_gaps": [
"thermal_memory_evidence_missing"
],
"health_reason": "timeout_threshold_exceeded",
"health_status": "degraded",
"latency_budget_exceeded": true,
"observation_scope": "single_runtime_result",
"production_cancellation": false,
"recommended_action": "review_latency_budget_or_degrade",
"retryable": true,
"risk_labels": [
"runtime_timeout_observed",
"latency_budget_exceeded"
],
"scheduler_owner": "orchestrator",
"schema_version": "inferedge-runtime-operation-summary-v1",
"thermal_memory_evidence_available": false,
"timeout_observed": true
},
"target": {
"board_name": "fixture-local-dev-machine",
"os": "local",
"runtime_tags": [
"entrypoint",
"runtime-operation-evidence"
],
"target_name": "entrypoint-local-fixture",
"target_type": "local"
}
}
87 changes: 87 additions & 0 deletions scripts/smoke_agent_runtime_edgeenv_preservation.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/usr/bin/env bash
set -euo pipefail

LAB_DIR="${LAB_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
OUTPUT_DIR="${OUTPUT_DIR:-$LAB_DIR/reports/agent_runtime_edgeenv_preservation}"

usage() {
cat <<'EOF'
InferEdge Agent Runtime EdgeEnv preservation smoke

Usage:
bash scripts/smoke_agent_runtime_edgeenv_preservation.sh [--output-dir <path>]
bash scripts/smoke_agent_runtime_edgeenv_preservation.sh --help

This smoke uses committed lightweight fixtures to verify that an EdgeEnv
runs-show artifact remains visible in the Lab-owned Agent Runtime report.
It does not require a live Jetson device and does not claim production
runtime operation.
EOF
}

while [[ $# -gt 0 ]]; do
case "$1" in
--help|-h)
usage
exit 0
;;
--output-dir)
if [[ $# -lt 2 ]]; then
echo "--output-dir requires a value" >&2
exit 2
fi
OUTPUT_DIR="$2"
shift
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
shift
done

cd "$LAB_DIR"
mkdir -p "$OUTPUT_DIR"

if command -v poetry >/dev/null 2>&1; then
LAB_CMD=(poetry run inferedgelab)
else
LAB_CMD=(inferedgelab)
fi

REPORT_JSON="$OUTPUT_DIR/agent_runtime_edgeenv_preservation.json"
REPORT_MD="$OUTPUT_DIR/agent_runtime_edgeenv_preservation.md"

echo "== Agent Runtime EdgeEnv preservation smoke =="
echo "Output: $OUTPUT_DIR"

"${LAB_CMD[@]}" agent-runtime-report \
--orchestration-summary examples/agent_runtime/agent_3_orchestration_summary.json \
--guard-analysis examples/agent_runtime/aiguard_runtime_guard_analysis.json \
--edgeenv-run-show examples/agent_runtime/edgeenv_run_show_runtime_operation.json \
--format json \
--output "$REPORT_JSON"

"${LAB_CMD[@]}" agent-runtime-report \
--orchestration-summary examples/agent_runtime/agent_3_orchestration_summary.json \
--guard-analysis examples/agent_runtime/aiguard_runtime_guard_analysis.json \
--edgeenv-run-show examples/agent_runtime/edgeenv_run_show_runtime_operation.json \
--format markdown \
--output "$REPORT_MD"

grep -q "edgeenv_preservation_context" "$REPORT_JSON"
grep -q "run-fixture-edgeenv-operation-0001" "$REPORT_JSON"
grep -q "inferedge-runtime-operation-summary-v1" "$REPORT_JSON"
grep -q "supplemental_evidence_not_gate" "$REPORT_JSON"
grep -q "Runtime Intelligence EdgeEnv Preservation" "$REPORT_MD"
grep -q "edgeenv_run_id" "$REPORT_MD"
grep -q "run-fixture-edgeenv-operation-0001" "$REPORT_MD"
grep -q "runtime_operation_health_reason" "$REPORT_MD"
grep -q "timeout_threshold_exceeded" "$REPORT_MD"
grep -q "review_latency_budget_or_degrade" "$REPORT_MD"
grep -q "supplemental_evidence_not_gate" "$REPORT_MD"
grep -q "Lab remains the final deployment decision owner" "$REPORT_MD"

echo "Agent Runtime EdgeEnv preservation smoke passed."
63 changes: 63 additions & 0 deletions tests/test_agent_runtime_edgeenv_preservation_smoke_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import subprocess
from pathlib import Path


REPO_ROOT = Path(__file__).resolve().parents[1]
SCRIPT = REPO_ROOT / "scripts" / "smoke_agent_runtime_edgeenv_preservation.sh"


def test_agent_runtime_edgeenv_preservation_smoke_script_help():
result = subprocess.run(
["bash", str(SCRIPT), "--help"],
check=False,
cwd=str(REPO_ROOT),
text=True,
capture_output=True,
)

assert result.returncode == 0
assert "Agent Runtime EdgeEnv preservation smoke" in result.stdout
assert "does not require a live Jetson device" in result.stdout


def test_agent_runtime_edgeenv_preservation_smoke_script_rejects_missing_output_dir_value():
result = subprocess.run(
["bash", str(SCRIPT), "--output-dir"],
check=False,
cwd=str(REPO_ROOT),
text=True,
capture_output=True,
)

assert result.returncode == 2
assert "--output-dir requires a value" in result.stderr


def test_agent_runtime_edgeenv_preservation_smoke_script_runs_gate(tmp_path):
output_dir = tmp_path / "agent_runtime_edgeenv_preservation"

result = subprocess.run(
["bash", str(SCRIPT), "--output-dir", str(output_dir)],
check=False,
cwd=str(REPO_ROOT),
text=True,
capture_output=True,
)

assert result.returncode == 0, result.stdout + result.stderr
assert "Agent Runtime EdgeEnv preservation smoke passed." in result.stdout

report_json = output_dir / "agent_runtime_edgeenv_preservation.json"
report_md = output_dir / "agent_runtime_edgeenv_preservation.md"
assert report_json.is_file()
assert report_md.is_file()

json_text = report_json.read_text(encoding="utf-8")
assert "edgeenv_preservation_context" in json_text
assert "run-fixture-edgeenv-operation-0001" in json_text

markdown = report_md.read_text(encoding="utf-8")
assert "Runtime Intelligence EdgeEnv Preservation" in markdown
assert "| edgeenv_run_id | run-fixture-edgeenv-operation-0001 |" in markdown
assert "| comparability_role | supplemental_evidence_not_gate |" in markdown
assert "Lab remains the final deployment decision owner" in markdown
9 changes: 9 additions & 0 deletions tests/test_agent_runtime_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -1514,11 +1514,20 @@ def test_agent_runtime_report_loads_committed_fixtures():
report = load_agent_runtime_reliability_bundle(
orchestration_summary_path="examples/agent_runtime/agent_3_orchestration_summary.json",
guard_analysis_path="examples/agent_runtime/aiguard_runtime_guard_analysis.json",
edgeenv_run_show_path=(
"examples/agent_runtime/edgeenv_run_show_runtime_operation.json"
),
)

assert report["agent_deployment_decision"]["decision"] == "blocked"
assert report["agent_runtime_summary"]["metrics"]["drop_rate"] == pytest.approx(14 / 24)
assert len(report["agent_runtime_summary"]["agents"]) == 3
context = report["agent_runtime_summary"]["edgeenv_preservation_context"]
assert context["run_id"] == "run-fixture-edgeenv-operation-0001"
assert context["runtime_operation_schema_version"] == (
"inferedge-runtime-operation-summary-v1"
)
assert context["comparability_role"] == "supplemental_evidence_not_gate"


def test_agent_runtime_report_surfaces_remote_execution_failure():
Expand Down
Loading