diff --git a/README.md b/README.md index 95ed0b6..f28ad57 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ poetry run inferedgelab agent-runtime-report \ --orchestration-summary examples/agent_runtime/agent_3_orchestration_summary.json \ --guard-analysis examples/agent_runtime/aiguard_runtime_guard_analysis.json bash scripts/smoke_agent_runtime_remote_paths.sh --output-dir reports/agent_runtime_remote_paths +bash scripts/smoke_agent_runtime_edgeenv_preservation.sh --output-dir reports/agent_runtime_edgeenv_preservation poetry run inferedgelab export-demo-evidence --output reports/studio_demo_evidence.md ``` @@ -188,7 +189,9 @@ Additional report paths: - The bundled agent evidence is a synthetic sustained high-load 3-agent scenario for local review. - Optional `--runtime-result ` input adds Runtime-side `runtime_health_snapshot`, `runtime_events`, and `runtime_operation_summary` context to the same Lab report. - Optional `--remote-dispatch ` input adds file-based worker selection, retry/fallback plan, and remote execution starter context when an Orchestrator `inferedge-remote-dispatch-result-v1` JSON is available. +- Optional `--edgeenv-run-show ` input adds EdgeEnv local run preservation context when an EdgeEnv `runs show` JSON is available. - `scripts/smoke_agent_runtime_remote_paths.sh` reproduces both committed remote dispatch starter paths: plan-only worker selection and fallback-recovered starter execution. +- `scripts/smoke_agent_runtime_edgeenv_preservation.sh` gates the committed EdgeEnv run-show fixture so the `Runtime Intelligence EdgeEnv Preservation` report section cannot disappear silently. - Remote dispatch remains starter evidence for local-first review; it does not claim production remote execution. Remote dispatch starter boundary: diff --git a/docs/portfolio/agent_runtime_reliability_report.md b/docs/portfolio/agent_runtime_reliability_report.md index 26e7853..6a34de9 100644 --- a/docs/portfolio/agent_runtime_reliability_report.md +++ b/docs/portfolio/agent_runtime_reliability_report.md @@ -29,6 +29,7 @@ Committed lightweight fixtures: - `examples/agent_runtime/aiguard_runtime_guard_analysis.json` - `examples/agent_runtime/remote_dispatch_plan_only.json` - `examples/agent_runtime/remote_dispatch_fallback_recovered.json` +- `examples/agent_runtime/edgeenv_run_show_runtime_operation.json` Generate a Markdown report: @@ -53,6 +54,20 @@ fallback-recovered starter execution. Both reports preserve `remote_runtime_summary_boundary=remote dispatch starter evidence only`; they do not start remote workers or claim production remote execution. +Reproduce the committed EdgeEnv run preservation report path: + +```bash +bash scripts/smoke_agent_runtime_edgeenv_preservation.sh \ + --output-dir reports/agent_runtime_edgeenv_preservation +``` + +The smoke verifies that the Lab-owned Markdown/JSON report keeps the +`Runtime Intelligence EdgeEnv Preservation` section, EdgeEnv run ID, Runtime +operation summary schema, and `comparability_role=supplemental_evidence_not_gate` +visible from a lightweight `runs show` fixture. This is local registry +preservation evidence, not a production telemetry database or deployment +decision override. + Current starter coverage: - file-based worker registry and task request ingestion diff --git a/examples/agent_runtime/edgeenv_run_show_runtime_operation.json b/examples/agent_runtime/edgeenv_run_show_runtime_operation.json new file mode 100644 index 0000000..dd37934 --- /dev/null +++ b/examples/agent_runtime/edgeenv_run_show_runtime_operation.json @@ -0,0 +1,69 @@ +{ + "created_at": "2026-05-29T09:47:14.594755Z", + "metrics": { + "latency_mean_ms": 28.0, + "latency_p50_ms": 28.0, + "latency_p95_ms": 36.0, + "latency_p99_ms": 36.0, + "throughput_fps": 30.0 + }, + "model": { + "model_format": "runtime-result-json", + "model_hash": "acb5f50c195d2feef052cb483fb8b3746ea8cd64a28aa8a44e9dfe2358b3c487", + "model_name": "entrypoint-runtime-operation-result", + "model_path": "examples/agent_runtime/runtime_operation_result.json", + "model_version": "v1" + }, + "protocol": { + "batch_size": 1, + "include_postprocess": false, + "include_preprocess": false, + "input_dtype": "json", + "input_shape": [ + 1 + ], + "precision": "evidence", + "repeat_runs": 1, + "task": "runtime-operation-evidence", + "warmup_runs": 0 + }, + "result_path": "examples/agent_runtime/.edgeenv/runs/run-fixture-edgeenv-operation-0001/result.json", + "run_id": "run-fixture-edgeenv-operation-0001", + "runtime": { + "execution_provider": "local-python", + "runtime": "entrypoint-smoke" + }, + "runtime_operation_summary": { + "deadline_missed": false, + "decision_owner": "lab", + "error_category": "runtime_timeout_observed", + "evidence_gaps": [ + "thermal_memory_evidence_missing" + ], + "health_reason": "timeout_threshold_exceeded", + "health_status": "degraded", + "latency_budget_exceeded": true, + "observation_scope": "single_runtime_result", + "production_cancellation": false, + "recommended_action": "review_latency_budget_or_degrade", + "retryable": true, + "risk_labels": [ + "runtime_timeout_observed", + "latency_budget_exceeded" + ], + "scheduler_owner": "orchestrator", + "schema_version": "inferedge-runtime-operation-summary-v1", + "thermal_memory_evidence_available": false, + "timeout_observed": true + }, + "target": { + "board_name": "fixture-local-dev-machine", + "os": "local", + "runtime_tags": [ + "entrypoint", + "runtime-operation-evidence" + ], + "target_name": "entrypoint-local-fixture", + "target_type": "local" + } +} diff --git a/scripts/smoke_agent_runtime_edgeenv_preservation.sh b/scripts/smoke_agent_runtime_edgeenv_preservation.sh new file mode 100755 index 0000000..c6d49ce --- /dev/null +++ b/scripts/smoke_agent_runtime_edgeenv_preservation.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +LAB_DIR="${LAB_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +OUTPUT_DIR="${OUTPUT_DIR:-$LAB_DIR/reports/agent_runtime_edgeenv_preservation}" + +usage() { + cat <<'EOF' +InferEdge Agent Runtime EdgeEnv preservation smoke + +Usage: + bash scripts/smoke_agent_runtime_edgeenv_preservation.sh [--output-dir ] + bash scripts/smoke_agent_runtime_edgeenv_preservation.sh --help + +This smoke uses committed lightweight fixtures to verify that an EdgeEnv +runs-show artifact remains visible in the Lab-owned Agent Runtime report. +It does not require a live Jetson device and does not claim production +runtime operation. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --help|-h) + usage + exit 0 + ;; + --output-dir) + if [[ $# -lt 2 ]]; then + echo "--output-dir requires a value" >&2 + exit 2 + fi + OUTPUT_DIR="$2" + shift + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done + +cd "$LAB_DIR" +mkdir -p "$OUTPUT_DIR" + +if command -v poetry >/dev/null 2>&1; then + LAB_CMD=(poetry run inferedgelab) +else + LAB_CMD=(inferedgelab) +fi + +REPORT_JSON="$OUTPUT_DIR/agent_runtime_edgeenv_preservation.json" +REPORT_MD="$OUTPUT_DIR/agent_runtime_edgeenv_preservation.md" + +echo "== Agent Runtime EdgeEnv preservation smoke ==" +echo "Output: $OUTPUT_DIR" + +"${LAB_CMD[@]}" agent-runtime-report \ + --orchestration-summary examples/agent_runtime/agent_3_orchestration_summary.json \ + --guard-analysis examples/agent_runtime/aiguard_runtime_guard_analysis.json \ + --edgeenv-run-show examples/agent_runtime/edgeenv_run_show_runtime_operation.json \ + --format json \ + --output "$REPORT_JSON" + +"${LAB_CMD[@]}" agent-runtime-report \ + --orchestration-summary examples/agent_runtime/agent_3_orchestration_summary.json \ + --guard-analysis examples/agent_runtime/aiguard_runtime_guard_analysis.json \ + --edgeenv-run-show examples/agent_runtime/edgeenv_run_show_runtime_operation.json \ + --format markdown \ + --output "$REPORT_MD" + +grep -q "edgeenv_preservation_context" "$REPORT_JSON" +grep -q "run-fixture-edgeenv-operation-0001" "$REPORT_JSON" +grep -q "inferedge-runtime-operation-summary-v1" "$REPORT_JSON" +grep -q "supplemental_evidence_not_gate" "$REPORT_JSON" +grep -q "Runtime Intelligence EdgeEnv Preservation" "$REPORT_MD" +grep -q "edgeenv_run_id" "$REPORT_MD" +grep -q "run-fixture-edgeenv-operation-0001" "$REPORT_MD" +grep -q "runtime_operation_health_reason" "$REPORT_MD" +grep -q "timeout_threshold_exceeded" "$REPORT_MD" +grep -q "review_latency_budget_or_degrade" "$REPORT_MD" +grep -q "supplemental_evidence_not_gate" "$REPORT_MD" +grep -q "Lab remains the final deployment decision owner" "$REPORT_MD" + +echo "Agent Runtime EdgeEnv preservation smoke passed." diff --git a/tests/test_agent_runtime_edgeenv_preservation_smoke_script.py b/tests/test_agent_runtime_edgeenv_preservation_smoke_script.py new file mode 100644 index 0000000..a7e5698 --- /dev/null +++ b/tests/test_agent_runtime_edgeenv_preservation_smoke_script.py @@ -0,0 +1,63 @@ +import subprocess +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "smoke_agent_runtime_edgeenv_preservation.sh" + + +def test_agent_runtime_edgeenv_preservation_smoke_script_help(): + result = subprocess.run( + ["bash", str(SCRIPT), "--help"], + check=False, + cwd=str(REPO_ROOT), + text=True, + capture_output=True, + ) + + assert result.returncode == 0 + assert "Agent Runtime EdgeEnv preservation smoke" in result.stdout + assert "does not require a live Jetson device" in result.stdout + + +def test_agent_runtime_edgeenv_preservation_smoke_script_rejects_missing_output_dir_value(): + result = subprocess.run( + ["bash", str(SCRIPT), "--output-dir"], + check=False, + cwd=str(REPO_ROOT), + text=True, + capture_output=True, + ) + + assert result.returncode == 2 + assert "--output-dir requires a value" in result.stderr + + +def test_agent_runtime_edgeenv_preservation_smoke_script_runs_gate(tmp_path): + output_dir = tmp_path / "agent_runtime_edgeenv_preservation" + + result = subprocess.run( + ["bash", str(SCRIPT), "--output-dir", str(output_dir)], + check=False, + cwd=str(REPO_ROOT), + text=True, + capture_output=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "Agent Runtime EdgeEnv preservation smoke passed." in result.stdout + + report_json = output_dir / "agent_runtime_edgeenv_preservation.json" + report_md = output_dir / "agent_runtime_edgeenv_preservation.md" + assert report_json.is_file() + assert report_md.is_file() + + json_text = report_json.read_text(encoding="utf-8") + assert "edgeenv_preservation_context" in json_text + assert "run-fixture-edgeenv-operation-0001" in json_text + + markdown = report_md.read_text(encoding="utf-8") + assert "Runtime Intelligence EdgeEnv Preservation" in markdown + assert "| edgeenv_run_id | run-fixture-edgeenv-operation-0001 |" in markdown + assert "| comparability_role | supplemental_evidence_not_gate |" in markdown + assert "Lab remains the final deployment decision owner" in markdown diff --git a/tests/test_agent_runtime_report.py b/tests/test_agent_runtime_report.py index 9a1f101..00d3470 100644 --- a/tests/test_agent_runtime_report.py +++ b/tests/test_agent_runtime_report.py @@ -1514,11 +1514,20 @@ def test_agent_runtime_report_loads_committed_fixtures(): report = load_agent_runtime_reliability_bundle( orchestration_summary_path="examples/agent_runtime/agent_3_orchestration_summary.json", guard_analysis_path="examples/agent_runtime/aiguard_runtime_guard_analysis.json", + edgeenv_run_show_path=( + "examples/agent_runtime/edgeenv_run_show_runtime_operation.json" + ), ) assert report["agent_deployment_decision"]["decision"] == "blocked" assert report["agent_runtime_summary"]["metrics"]["drop_rate"] == pytest.approx(14 / 24) assert len(report["agent_runtime_summary"]["agents"]) == 3 + context = report["agent_runtime_summary"]["edgeenv_preservation_context"] + assert context["run_id"] == "run-fixture-edgeenv-operation-0001" + assert context["runtime_operation_schema_version"] == ( + "inferedge-runtime-operation-summary-v1" + ) + assert context["comparability_role"] == "supplemental_evidence_not_gate" def test_agent_runtime_report_surfaces_remote_execution_failure():